Skip to content

Commit 10c4a55

Browse files
justheuristicjustheuristic
committed
Merge branch 'fall19' of github.com:yandexdataschool/practical_dl into fall19
2 parents ae91ee1 + 4a483a5 commit 10c4a55

File tree

1 file changed

+97
-112
lines changed

1 file changed

+97
-112
lines changed

‎week08_autoencoders/autoencoders_torch.ipynb

Lines changed: 97 additions & 112 deletions
Original file line numberDiff line numberDiff line change
@@ -222,9 +222,9 @@
222222
" self.enc = nn.Sequential(View(-1, np.prod(img_shape)), nn.Linear(np.prod(img_shape), code_size))\n",
223223
" self.dec = nn.Sequential(nn.Linear(code_size, np.prod(img_shape)), View(-1, img_shape[0], img_shape[1], img_shape[2])) \n",
224224
" \n",
225-
" def batch_loss(self, batch):\n",
225+
" def batch_loss(self, batch, reference):\n",
226226
" reconstruction = #<Your code: define reconstruction object>\n",
227-
" return torch.mean((batch - reconstruction)**2)\n",
227+
" return torch.mean((reference - reconstruction)**2)\n",
228228
" "
229229
]
230230
},
@@ -245,21 +245,43 @@
245245
"metadata": {},
246246
"outputs": [],
247247
"source": [
248-
"def train(model, dataset, num_epoch=32):\n",
248+
"from tqdm import tqdm\n",
249+
"def train(model, dataset, dataset_test, num_epoch=32, gd=None, noise_function=None, noise_function_params=None):\n",
249250
" model.double()\n",
250251
" model.to(device)\n",
251-
" gd = optim.Adamax(model.parameters(), lr=0.002)\n",
252+
" if gd is None:\n",
253+
" gd = optim.Adamax(model.parameters(), lr=0.002)\n",
254+
" if noise_function_params is None:\n",
255+
" noise_function_params = {}\n",
256+
" \n",
252257
" dataloader = DataLoader(dataset, batch_size=BATCH_SIZE, shuffle=True)\n",
253258
" losses = []\n",
259+
" dataloader_test = DataLoader(dataset_test, batch_size=BATCH_SIZE, shuffle=True)\n",
260+
" scores = []\n",
261+
"\n",
254262
" for epoch in range(num_epoch):\n",
255-
" for i, (batch) in enumerate(dataloader):\n",
263+
" model.train(True)\n",
264+
" for i, (batch) in tqdm(enumerate(dataloader)):\n",
256265
" gd.zero_grad()\n",
257-
" loss = model.batch_loss(batch.to(device=device))\n",
266+
" if noise_function is not None:\n",
267+
" batch_noised = noise_function(batch, **noise_function_params).to(device=device)\n",
268+
" loss = model.batch_loss(batch_noised, batch.to(device=device))\n",
269+
" else:\n",
270+
" batch = batch.to(device=device)\n",
271+
" loss = model.batch_loss(batch, batch)\n",
258272
" (loss).backward()\n",
259273
" losses.append(loss.data.cpu().numpy())\n",
260274
" gd.step()\n",
261275
" gd.zero_grad()\n",
262-
" print(\"#%i, Train loss: %.7f\"%(epoch+1,np.mean(losses)),flush=True)"
276+
" train_mse = np.mean(losses[-(i+1):])\n",
277+
" \n",
278+
" model.train(False)\n",
279+
" for i, (batch) in enumerate(dataloader_test):\n",
280+
" batch = batch.to(device=device)\n",
281+
" scores.append(model.batch_loss(batch, batch).data.cpu().numpy())\n",
282+
" test_mse = np.mean(scores[-(i+1):])\n",
283+
"\n",
284+
" print(f\"{epoch+1}, Train loss: {train_mse}, Test loss: {test_mse}\")"
263285
]
264286
},
265287
{
@@ -270,7 +292,9 @@
270292
"source": [
271293
"def visualize(img, model):\n",
272294
" \"\"\"Draws original, encoded and decoded images\"\"\"\n",
295+
" model.train(False)\n",
273296
" code = model.enc(img[None].cuda(device = device))\n",
297+
"\n",
274298
" reco = model.dec(code)\n",
275299
"\n",
276300
" plt.subplot(1,3,1)\n",
@@ -341,7 +365,7 @@
341365
],
342366
"source": [
343367
"aenc = pca_autoencoder()\n",
344-
"train(aenc, X_train_tensor, 40)"
368+
"train(aenc, X_train_tensor, X_test_tensor, 40)"
345369
]
346370
},
347371
{
@@ -361,7 +385,8 @@
361385
"dataloader_test = DataLoader(X_test_tensor, batch_size=BATCH_SIZE, shuffle=True)\n",
362386
"scores = []\n",
363387
"for i, (batch) in enumerate(dataloader_test):\n",
364-
" scores.append(aenc.batch_loss(batch.cuda(device = device)).data.cpu().numpy())\n",
388+
" batch = batch.to(device=device)\n",
389+
" scores.append(aenc.batch_loss(batch, batch).data.cpu().numpy())\n",
365390
"print (np.mean(scores))"
366391
]
367392
},
@@ -468,10 +493,56 @@
468493
" self.enc = #<Your code: define encoder as per instructions above>\n",
469494
" self.dec = #<Your code: define decoder as per instructions above>\n",
470495
" \n",
471-
" def batch_loss(self, batch):\n",
496+
" def batch_loss(self, batch, reference):\n",
472497
" a = self.enc(batch)\n",
473498
" reconstruction = self.dec(a)\n",
474-
" return torch.mean((batch - reconstruction)**2)"
499+
" return torch.mean((reference - reconstruction)**2)"
500+
]
501+
},
502+
{
503+
"cell_type": "code",
504+
"execution_count": 53,
505+
"metadata": {},
506+
"outputs": [
507+
{
508+
"name": "stdout",
509+
"output_type": "stream",
510+
"text": [
511+
"Testing code size 1\n",
512+
"Testing code size 8\n",
513+
"Testing code size 32\n",
514+
"Testing code size 128\n",
515+
"Testing code size 512\n",
516+
"Testing code size 1024\n",
517+
"All tests passed!\n"
518+
]
519+
}
520+
],
521+
"source": [
522+
"#Check autoencoder shapes along different code_sizes\n",
523+
"get_dim = lambda layer: np.prod(layer.output_shape[1:])\n",
524+
"for code_size in [1,8,32,128,512,1024]:\n",
525+
" help_tensor = next(iter(DataLoader(X_train_tensor, batch_size=BATCH_SIZE)))\n",
526+
" model = pca_autoencoder_deep(code_size).double().to(device)\n",
527+
" encoder_out = model.enc(help_tensor.cuda(device))\n",
528+
" decoder_out = model.dec(encoder_out)\n",
529+
" print(\"Testing code size %i\" % code_size)\n",
530+
"\n",
531+
" assert encoder_out.shape[1:]==torch.Size([code_size]),\"encoder must output a code of required size\"\n",
532+
" assert decoder_out.shape[1:]==img_shape, \"decoder must output an image of valid shape\"\n",
533+
"\n",
534+
" assert (len(list(model.dec.children())) >= 6), \"decoder must contain at least 3 dense layers\"\n",
535+
"\n",
536+
"print(\"All tests passed!\")"
537+
]
538+
},
539+
{
540+
"cell_type": "markdown",
541+
"metadata": {},
542+
"source": [
543+
"__Hint:__ if you're getting \"Encoder layer is smaller than bottleneck\" error, use code_size when defining intermediate layers. \n",
544+
"\n",
545+
"For example, such layer may have code_size*2 units."
475546
]
476547
},
477548
{
@@ -538,7 +609,7 @@
538609
],
539610
"source": [
540611
"aenc_deep = pca_autoencoder_deep()\n",
541-
"train(aenc_deep, X_train_tensor, 50)"
612+
"train(aenc_deep, X_train_tensor, X_test_tensor, 50)"
542613
]
543614
},
544615
{
@@ -548,52 +619,6 @@
548619
"Training may take long, it's okay."
549620
]
550621
},
551-
{
552-
"cell_type": "code",
553-
"execution_count": 53,
554-
"metadata": {},
555-
"outputs": [
556-
{
557-
"name": "stdout",
558-
"output_type": "stream",
559-
"text": [
560-
"Testing code size 1\n",
561-
"Testing code size 8\n",
562-
"Testing code size 32\n",
563-
"Testing code size 128\n",
564-
"Testing code size 512\n",
565-
"Testing code size 1024\n",
566-
"All tests passed!\n"
567-
]
568-
}
569-
],
570-
"source": [
571-
"#Check autoencoder shapes along different code_sizes\n",
572-
"get_dim = lambda layer: np.prod(layer.output_shape[1:])\n",
573-
"for code_size in [1,8,32,128,512,1024]:\n",
574-
" help_tensor = next(iter(DataLoader(X_train_tensor, batch_size=BATCH_SIZE)))\n",
575-
" model = pca_autoencoder_deep(code_size).to(device)\n",
576-
" encoder_out = model.enc(help_tensor.cuda(device))\n",
577-
" decoder_out = model.dec(encoder_out)\n",
578-
" print(\"Testing code size %i\" % code_size)\n",
579-
"\n",
580-
" assert encoder_out.shape[1:]==torch.Size([code_size]),\"encoder must output a code of required size\"\n",
581-
" assert decoder_out.shape[1:]==img_shape, \"decoder must output an image of valid shape\"\n",
582-
"\n",
583-
" assert (len(list(model.dec.children())) >= 6), \"decoder must contain at least 3 dense layers\"\n",
584-
"\n",
585-
"print(\"All tests passed!\")"
586-
]
587-
},
588-
{
589-
"cell_type": "markdown",
590-
"metadata": {},
591-
"source": [
592-
"__Hint:__ if you're getting \"Encoder layer is smaller than bottleneck\" error, use code_size when defining intermediate layers. \n",
593-
"\n",
594-
"For example, such layer may have code_size*2 units."
595-
]
596-
},
597622
{
598623
"cell_type": "code",
599624
"execution_count": 55,
@@ -668,11 +693,13 @@
668693
}
669694
],
670695
"source": [
696+
"aenc_deep.train(False)\n",
671697
"dataloader_test = DataLoader(X_test_tensor, batch_size=BATCH_SIZE, shuffle=True)\n",
672698
"scores = []\n",
673699
"for i, (batch) in enumerate(dataloader_test):\n",
674-
" scores.append(aenc_deep.batch_loss(batch.cuda(device = device)).data.cpu().numpy())\n",
675-
" encoder_out = aenc_deep.enc(batch.cuda(device = device))\n",
700+
" batch = batch.to(device=device)\n",
701+
" scores.append(aenc_deep.batch_loss(batch, batch).data.cpu().numpy())\n",
702+
" encoder_out = aenc_deep.enc(batch)\n",
676703
"reconstruction_mse = np.mean(scores)\n",
677704
"\n",
678705
"assert reconstruction_mse <= 0.0055, \"Compression is too lossy. See tips below.\"\n",
@@ -774,54 +801,11 @@
774801
"plt.subplot(1,4,1)\n",
775802
"plt.imshow(X[0].transpose([1,2,0]))\n",
776803
"plt.subplot(1,4,2)\n",
777-
"plt.imshow(apply_gaussian_noise(X[:1],sigma=0.01)[0].transpose([1,2,0]).clip(0, 1))\n",
804+
"plt.imshow(apply_gaussian_noise(X[:1],sigma=0.01).data.numpy()[0].transpose([1,2,0]).clip(0, 1))\n",
778805
"plt.subplot(1,4,3)\n",
779-
"plt.imshow(apply_gaussian_noise(X[:1],sigma=0.1)[0].transpose([1,2,0]).clip(0, 1))\n",
806+
"plt.imshow(apply_gaussian_noise(X[:1],sigma=0.1).data.numpy()[0].transpose([1,2,0]).clip(0, 1))\n",
780807
"plt.subplot(1,4,4)\n",
781-
"plt.imshow(apply_gaussian_noise(X[:1],sigma=0.5)[0].transpose([1,2,0]).clip(0, 1))"
782-
]
783-
},
784-
{
785-
"cell_type": "code",
786-
"execution_count": 166,
787-
"metadata": {},
788-
"outputs": [],
789-
"source": [
790-
"def train_noise(model, dataset, num_epoch=50):\n",
791-
" model.double()\n",
792-
" model.to(device)\n",
793-
" gd = optim.Adamax(model.parameters(), lr=0.002)\n",
794-
" dataloader = DataLoader(dataset, batch_size=BATCH_SIZE, shuffle=True)\n",
795-
" losses = []\n",
796-
" for epoch in range(num_epoch):\n",
797-
" for i, (batch) in enumerate(dataloader):\n",
798-
" gd.zero_grad()\n",
799-
" loss = model.batch_loss(batch.cuda(device=device))\n",
800-
" (loss).backward()\n",
801-
" losses.append(loss.data.cpu().numpy())\n",
802-
" gd.step()\n",
803-
" gd.zero_grad()\n",
804-
" print(\"#%i, Train loss: %.7f\"%(epoch + 1, np.mean(losses)), flush=True)"
805-
]
806-
},
807-
{
808-
"cell_type": "code",
809-
"execution_count": 167,
810-
"metadata": {},
811-
"outputs": [],
812-
"source": [
813-
"X_train_noise = apply_gaussian_noise(X_train)\n",
814-
"X_test_noise = apply_gaussian_noise(X_test)"
815-
]
816-
},
817-
{
818-
"cell_type": "code",
819-
"execution_count": 161,
820-
"metadata": {},
821-
"outputs": [],
822-
"source": [
823-
"X_train_tensor_n = torch.from_numpy(X_train_noise).type(torch.DoubleTensor)\n",
824-
"X_test_tensor_n = torch.Tensor(X_test_noise).type(torch.DoubleTensor)"
808+
"plt.imshow(apply_gaussian_noise(X[:1],sigma=0.5).data.numpy()[0].transpose([1,2,0]).clip(0, 1))"
825809
]
826810
},
827811
{
@@ -888,7 +872,7 @@
888872
],
889873
"source": [
890874
"aenc = pca_autoencoder()\n",
891-
"train(aenc, X_train_tensor_n, 50)"
875+
"train(aenc, X_train_tensor, X_test_tensor, 50, noise_function=apply_gaussian_noise)"
892876
]
893877
},
894878
{
@@ -974,16 +958,17 @@
974958
}
975959
],
976960
"source": [
977-
"dataloader_test = DataLoader(X_test_tensor_n, batch_size=BATCH_SIZE, shuffle=True)\n",
961+
"dataloader_test = DataLoader(X_test_tensor, batch_size=BATCH_SIZE, shuffle=True)\n",
978962
"scores = []\n",
979963
"for i, (batch) in enumerate(dataloader_test):\n",
980-
" scores.append(aenc.batch_loss(batch.cuda(device = device)).data.cpu().numpy())\n",
981-
" encoder_out = aenc.enc(batch.cuda(device = device))\n",
964+
" batch_noised = apply_gaussian_noise(batch).to(device=device)\n",
965+
" scores.append(aenc.batch_loss(batch_noised, batch.cuda(device = device)).data.cpu().numpy())\n",
966+
" encoder_out = aenc.enc(batch_noised)\n",
982967
"reconstruction_mse = np.mean(scores)\n",
983968
"\n",
984969
"print(\"Final MSE:\", reconstruction_mse)\n",
985970
"for i in range(5):\n",
986-
" img = X_test_tensor_n[i]\n",
971+
" img = apply_gaussian_noise(X_test_tensor[i])\n",
987972
" visualize(img,aenc)"
988973
]
989974
},
@@ -1365,7 +1350,7 @@
13651350
"name": "python",
13661351
"nbconvert_exporter": "python",
13671352
"pygments_lexer": "ipython3",
1368-
"version": "3.6.8"
1353+
"version": "3.7.3"
13691354
}
13701355
},
13711356
"nbformat": 4,

0 commit comments

Comments
 (0)