From 5c26322136a71c3829c19ad27e73d716ddc0d630 Mon Sep 17 00:00:00 2001 From: katsunori waragai Date: Thu, 15 Jun 2023 17:17:59 +0900 Subject: [PATCH 01/11] converted from notbeook --- style_transfer.py | 240 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 240 insertions(+) create mode 100644 style_transfer.py diff --git a/style_transfer.py b/style_transfer.py new file mode 100644 index 0000000..b795094 --- /dev/null +++ b/style_transfer.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python +# coding: utf-8 + +# In[ ]: + + +from google.colab import drive +drive.mount('/content/drive/') + + +# ls + +# In[ ]: + + +get_ipython().system('pip install torch torchvision') + + +# In[ ]: + + +#Doing this because, sometimes we get an error 'module 'PIL.Image' has no attribute 'register_extensions' in Google Colab +#uninstall the old one +get_ipython().system('pip uninstall -y Pillow') +# install the new one +get_ipython().system('pip install Pillow==4.1.1') + + +# In[ ]: + + +#importing libraries +import numpy as np +import matplotlib.pyplot as plt +from PIL import Image + +import torch +from torchvision import transforms, models + + +# In[ ]: + + +vgg = models.vgg19(pretrained=True).features + +for param in vgg.parameters(): + param.requires_grad_(False) + + +# In[ ]: + + +device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') +print('Cuda Available: ', torch.cuda.is_available()) +vgg.to(device) + + +# In[ ]: + + +def load_image(path, max_size=400, shape=None): + image = Image.open(path).convert('RGB') + + if max(image.size) > max_size: + size = max_size + else: + size = max(image.size) + if shape is not None: + size = shape + + in_transform = transforms.Compose([ + transforms.Resize(size), + transforms.ToTensor(), + transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)) + ]) + + image = in_transform(image)[:3,:,:].unsqueeze(0) + + return image + + + +# In[ ]: + + +#load content image +content = load_image('drive/My Drive/Colab Notebooks/Deep Learning/PyTorch Scholarship/Style Transfer/assets/surya2.jpg').to(device) + +#load style image +style = load_image('drive/My Drive/Colab Notebooks/Deep Learning/PyTorch Scholarship/Style Transfer/assets/oily_mcoilface.jpg', shape=content.shape[-2:]).to(device) + + +# In[ ]: + + +#function to convert image from it's normalised form to back to regular form +def imconvert(tensor): + tensor = tensor.cpu().clone().detach() + tensor = tensor.numpy().squeeze() + tensor = tensor.transpose(1,2,0) + tensor = tensor * np.array((0.229, 0.224, 0.225)) + np.array((0.485, 0.456, 0.406)) + tensor = tensor.clip(0,1) + return tensor + + +# In[ ]: + + +fig = plt.figure(figsize=(20,10)) +ax1 = fig.add_subplot(1,2,1, xticks=[], yticks=[]) +ax1.imshow(imconvert(content)) +ax2 = fig.add_subplot(1,2,2, xticks=[], yticks=[]) +ax2.imshow(imconvert(style)) + + +# In[ ]: + + +#printing the vgg model +vgg + + +# In[ ]: + + +#defining the function to get layers +def get_features(image, model, layers=None): + if layers is None: + layers = {'0':'conv1_1','5':'conv2_1', '10':'conv3_1', '19':'conv4_1', '28':'conv5_1', '21':'conv4_2'} + + features = {} + for name, layer in model._modules.items(): + image = layer(image) + if name in layers: + features[layers[name]]=image + + return features + + +# In[ ]: + + +#calculating the gram matrix +def gram_matrix(tensor): + batch_size, depth, height, width = tensor.shape + + tensor = tensor.view(depth, -1) + tensor = torch.mm(tensor, tensor.t()) + return tensor + + +# In[ ]: + + +style_features = get_features(style, vgg) +content_features = get_features(content, vgg) + +style_grams = {layer: gram_matrix(style_features[layer]) for layer in style_features} + +#we could start with random image, but it would be good to start with content image +target = content.clone().requires_grad_(True).to(device) + + +# In[ ]: + + +style_weights = {'conv1_1': 1., + 'conv2_1': 0.8, + 'conv3_1': 0.5, + 'conv4_1': 0.3, + 'conv5_1': 0.1} + + +content_weight = 1 # alpha +style_weight = 5e6 # beta + + +# In[ ]: + + +optimizer = torch.optim.Adam([target], lr=0.003) + +steps = 2400 +print_every = 400 + +for i in range(1,steps+1): + + target_features = get_features(target, vgg) + content_loss = torch.mean((content_features['conv4_2']-target_features['conv4_2'])**2) + + style_loss = 0 + for layer in style_weights: + + target_feature = target_features[layer] + + _, d, h, w = target_feature.shape + + target_gram = gram_matrix(target_feature) + + style_gram = style_grams[layer] + + layer_style_loss = style_weights[layer]*torch.mean((target_gram - style_gram)**2) + + style_loss += layer_style_loss/ (d*h*w) + + total_loss = style_weight*style_loss + content_weight*content_loss + + optimizer.zero_grad() + total_loss.backward() + optimizer.step() + + if i%print_every==0: + print('Total Loss: ', total_loss.item()) + plt.imshow(imconvert(target)) + + +# In[ ]: + + +fig = plt.figure(figsize=(22,10)) +ax1 = fig.add_subplot(1,3,1, xticks=[], yticks=[]) +ax1.imshow(imconvert(content)) +ax2 = fig.add_subplot(1,3,2, xticks=[], yticks=[]) +ax2.imshow(imconvert(style)) + +ax3 = fig.add_subplot(1,3,3, xticks=[], yticks=[]) +ax3.imshow(imconvert(target)) + + +# In[ ]: + + +plt.imsave('drive/My Drive/Colab Notebooks/Deep Learning/PyTorch Scholarship/Style Transfer/assets/surya2_style.jpg', imconvert(target)) + + +# In[ ]: + + + + From f1a445bb9340d10dd14bdf778aae206fbb255c69 Mon Sep 17 00:00:00 2001 From: katsunori waragai Date: Thu, 15 Jun 2023 17:20:12 +0900 Subject: [PATCH 02/11] converted from notebook --- style_transfer.py | 67 ----------------------------------------------- 1 file changed, 67 deletions(-) diff --git a/style_transfer.py b/style_transfer.py index b795094..91d487d 100644 --- a/style_transfer.py +++ b/style_transfer.py @@ -1,24 +1,9 @@ -#!/usr/bin/env python -# coding: utf-8 - -# In[ ]: - - from google.colab import drive drive.mount('/content/drive/') - -# ls - -# In[ ]: - - get_ipython().system('pip install torch torchvision') -# In[ ]: - - #Doing this because, sometimes we get an error 'module 'PIL.Image' has no attribute 'register_extensions' in Google Colab #uninstall the old one get_ipython().system('pip uninstall -y Pillow') @@ -26,8 +11,6 @@ get_ipython().system('pip install Pillow==4.1.1') -# In[ ]: - #importing libraries import numpy as np @@ -37,27 +20,16 @@ import torch from torchvision import transforms, models - -# In[ ]: - - vgg = models.vgg19(pretrained=True).features for param in vgg.parameters(): param.requires_grad_(False) -# In[ ]: - - device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') print('Cuda Available: ', torch.cuda.is_available()) vgg.to(device) - -# In[ ]: - - def load_image(path, max_size=400, shape=None): image = Image.open(path).convert('RGB') @@ -80,8 +52,6 @@ def load_image(path, max_size=400, shape=None): -# In[ ]: - #load content image content = load_image('drive/My Drive/Colab Notebooks/Deep Learning/PyTorch Scholarship/Style Transfer/assets/surya2.jpg').to(device) @@ -90,9 +60,6 @@ def load_image(path, max_size=400, shape=None): style = load_image('drive/My Drive/Colab Notebooks/Deep Learning/PyTorch Scholarship/Style Transfer/assets/oily_mcoilface.jpg', shape=content.shape[-2:]).to(device) -# In[ ]: - - #function to convert image from it's normalised form to back to regular form def imconvert(tensor): tensor = tensor.cpu().clone().detach() @@ -103,8 +70,6 @@ def imconvert(tensor): return tensor -# In[ ]: - fig = plt.figure(figsize=(20,10)) ax1 = fig.add_subplot(1,2,1, xticks=[], yticks=[]) @@ -112,17 +77,9 @@ def imconvert(tensor): ax2 = fig.add_subplot(1,2,2, xticks=[], yticks=[]) ax2.imshow(imconvert(style)) - -# In[ ]: - - #printing the vgg model vgg - -# In[ ]: - - #defining the function to get layers def get_features(image, model, layers=None): if layers is None: @@ -136,10 +93,6 @@ def get_features(image, model, layers=None): return features - -# In[ ]: - - #calculating the gram matrix def gram_matrix(tensor): batch_size, depth, height, width = tensor.shape @@ -149,9 +102,6 @@ def gram_matrix(tensor): return tensor -# In[ ]: - - style_features = get_features(style, vgg) content_features = get_features(content, vgg) @@ -161,9 +111,6 @@ def gram_matrix(tensor): target = content.clone().requires_grad_(True).to(device) -# In[ ]: - - style_weights = {'conv1_1': 1., 'conv2_1': 0.8, 'conv3_1': 0.5, @@ -175,9 +122,6 @@ def gram_matrix(tensor): style_weight = 5e6 # beta -# In[ ]: - - optimizer = torch.optim.Adam([target], lr=0.003) steps = 2400 @@ -214,9 +158,6 @@ def gram_matrix(tensor): plt.imshow(imconvert(target)) -# In[ ]: - - fig = plt.figure(figsize=(22,10)) ax1 = fig.add_subplot(1,3,1, xticks=[], yticks=[]) ax1.imshow(imconvert(content)) @@ -227,14 +168,6 @@ def gram_matrix(tensor): ax3.imshow(imconvert(target)) -# In[ ]: - - plt.imsave('drive/My Drive/Colab Notebooks/Deep Learning/PyTorch Scholarship/Style Transfer/assets/surya2_style.jpg', imconvert(target)) -# In[ ]: - - - - From 6965342a5522bd0aa3f2205cf670f72992f8c9c7 Mon Sep 17 00:00:00 2001 From: katsunori waragai Date: Thu, 15 Jun 2023 17:40:00 +0900 Subject: [PATCH 03/11] under develop --- requirements.txt | 3 +++ style_transfer.py | 30 ++++++++++++++++-------------- 2 files changed, 19 insertions(+), 14 deletions(-) create mode 100755 requirements.txt mode change 100644 => 100755 style_transfer.py diff --git a/requirements.txt b/requirements.txt new file mode 100755 index 0000000..82e4df1 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +torch +torchvision +Pillow==4.1.1 diff --git a/style_transfer.py b/style_transfer.py old mode 100644 new mode 100755 index 91d487d..d781708 --- a/style_transfer.py +++ b/style_transfer.py @@ -1,14 +1,14 @@ -from google.colab import drive -drive.mount('/content/drive/') +# from google.colab import drive +# drive.mount('/content/drive/') -get_ipython().system('pip install torch torchvision') - - -#Doing this because, sometimes we get an error 'module 'PIL.Image' has no attribute 'register_extensions' in Google Colab -#uninstall the old one -get_ipython().system('pip uninstall -y Pillow') -# install the new one -get_ipython().system('pip install Pillow==4.1.1') +# get_ipython().system('pip install torch torchvision') +# +# +# #Doing this because, sometimes we get an error 'module 'PIL.Image' has no attribute 'register_extensions' in Google Colab +# #uninstall the old one +# get_ipython().system('pip uninstall -y Pillow') +# # install the new one +# get_ipython().system('pip install Pillow==4.1.1') @@ -50,14 +50,16 @@ def load_image(path, max_size=400, shape=None): return image +from pathlib import Path +REPO_ROOT = Path("/media/waragai/ExtremeSSD/github/Image-Style-Transfer-Pytorch") - - +content_image = REPO_ROOT / "assets/surya2.jpg" #load content image -content = load_image('drive/My Drive/Colab Notebooks/Deep Learning/PyTorch Scholarship/Style Transfer/assets/surya2.jpg').to(device) +content = load_image(str(content_image)).to(device) +style_image = REPO_ROOT / "assets/oily_mcoilface.jpg" #load style image -style = load_image('drive/My Drive/Colab Notebooks/Deep Learning/PyTorch Scholarship/Style Transfer/assets/oily_mcoilface.jpg', shape=content.shape[-2:]).to(device) +style = load_image(str(style_image), shape=content.shape[-2:]).to(device) #function to convert image from it's normalised form to back to regular form From e2048cb091deabe21c1c74a199b5eab665f0ec10 Mon Sep 17 00:00:00 2001 From: katsunori waragai Date: Thu, 15 Jun 2023 17:47:53 +0900 Subject: [PATCH 04/11] under develop --- style_transfer.py | 105 +++++++++++++++++++++------------------------- 1 file changed, 48 insertions(+), 57 deletions(-) diff --git a/style_transfer.py b/style_transfer.py index d781708..383914c 100755 --- a/style_transfer.py +++ b/style_transfer.py @@ -1,18 +1,7 @@ -# from google.colab import drive -# drive.mount('/content/drive/') - -# get_ipython().system('pip install torch torchvision') -# -# -# #Doing this because, sometimes we get an error 'module 'PIL.Image' has no attribute 'register_extensions' in Google Colab -# #uninstall the old one -# get_ipython().system('pip uninstall -y Pillow') -# # install the new one -# get_ipython().system('pip install Pillow==4.1.1') - - #importing libraries +from pathlib import Path + import numpy as np import matplotlib.pyplot as plt from PIL import Image @@ -20,47 +9,27 @@ import torch from torchvision import transforms, models -vgg = models.vgg19(pretrained=True).features - -for param in vgg.parameters(): - param.requires_grad_(False) - - -device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') -print('Cuda Available: ', torch.cuda.is_available()) -vgg.to(device) +REPO_ROOT = Path(__file__).parent def load_image(path, max_size=400, shape=None): image = Image.open(path).convert('RGB') - + if max(image.size) > max_size: size = max_size else: size = max(image.size) if shape is not None: size = shape - + in_transform = transforms.Compose([ transforms.Resize(size), transforms.ToTensor(), transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)) ]) - - image = in_transform(image)[:3,:,:].unsqueeze(0) - - return image - -from pathlib import Path -REPO_ROOT = Path("/media/waragai/ExtremeSSD/github/Image-Style-Transfer-Pytorch") - -content_image = REPO_ROOT / "assets/surya2.jpg" -#load content image -content = load_image(str(content_image)).to(device) -style_image = REPO_ROOT / "assets/oily_mcoilface.jpg" -#load style image -style = load_image(str(style_image), shape=content.shape[-2:]).to(device) + image = in_transform(image)[:3, :, :].unsqueeze(0) + return image #function to convert image from it's normalised form to back to regular form def imconvert(tensor): @@ -72,38 +41,60 @@ def imconvert(tensor): return tensor - -fig = plt.figure(figsize=(20,10)) -ax1 = fig.add_subplot(1,2,1, xticks=[], yticks=[]) -ax1.imshow(imconvert(content)) -ax2 = fig.add_subplot(1,2,2, xticks=[], yticks=[]) -ax2.imshow(imconvert(style)) - -#printing the vgg model -vgg - -#defining the function to get layers +# defining the function to get layers def get_features(image, model, layers=None): if layers is None: - layers = {'0':'conv1_1','5':'conv2_1', '10':'conv3_1', '19':'conv4_1', '28':'conv5_1', '21':'conv4_2'} - + layers = {'0': 'conv1_1', '5': 'conv2_1', '10': 'conv3_1', '19': 'conv4_1', '28': 'conv5_1', '21': 'conv4_2'} + features = {} for name, layer in model._modules.items(): image = layer(image) if name in layers: - features[layers[name]]=image - + features[layers[name]] = image + return features -#calculating the gram matrix + +# calculating the gram matrix def gram_matrix(tensor): batch_size, depth, height, width = tensor.shape - + tensor = tensor.view(depth, -1) tensor = torch.mm(tensor, tensor.t()) return tensor +vgg = models.vgg19(pretrained=True).features + +for param in vgg.parameters(): + param.requires_grad_(False) + + +device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') +print('Cuda Available: ', torch.cuda.is_available()) +vgg.to(device) + + + +content_image = REPO_ROOT / "assets/surya2.jpg" +#load content image +content = load_image(str(content_image)).to(device) + +style_image = REPO_ROOT / "assets/oily_mcoilface.jpg" +#load style image +style = load_image(str(style_image), shape=content.shape[-2:]).to(device) + + +fig = plt.figure(figsize=(20,10)) +ax1 = fig.add_subplot(1,2,1, xticks=[], yticks=[]) +ax1.imshow(imconvert(content)) +ax2 = fig.add_subplot(1,2,2, xticks=[], yticks=[]) +ax2.imshow(imconvert(style)) + +#printing the vgg model +vgg + + style_features = get_features(style, vgg) content_features = get_features(content, vgg) @@ -169,7 +160,7 @@ def gram_matrix(tensor): ax3 = fig.add_subplot(1,3,3, xticks=[], yticks=[]) ax3.imshow(imconvert(target)) - -plt.imsave('drive/My Drive/Colab Notebooks/Deep Learning/PyTorch Scholarship/Style Transfer/assets/surya2_style.jpg', imconvert(target)) +converted_name = REPO_ROOT / "assets/surya2_style.jpg" +plt.imsave(str(converted_name), imconvert(target)) From 82bd0a50b7b641f00a9f58e55c91e5c554b28cfc Mon Sep 17 00:00:00 2001 From: katsunori waragai Date: Thu, 15 Jun 2023 18:19:44 +0900 Subject: [PATCH 05/11] began to work --- style_transfer.py | 62 ++++++++++++++++++++++++++--------------------- 1 file changed, 35 insertions(+), 27 deletions(-) diff --git a/style_transfer.py b/style_transfer.py index 383914c..360b2b6 100755 --- a/style_transfer.py +++ b/style_transfer.py @@ -5,11 +5,15 @@ import numpy as np import matplotlib.pyplot as plt from PIL import Image +# import cv2 +import skimage import torch from torchvision import transforms, models REPO_ROOT = Path(__file__).parent +DST_DIR = REPO_ROOT / "output" +DST_DIR.mkdir(exist_ok=True) def load_image(path, max_size=400, shape=None): image = Image.open(path).convert('RGB') @@ -84,12 +88,14 @@ def gram_matrix(tensor): #load style image style = load_image(str(style_image), shape=content.shape[-2:]).to(device) +print("loaded images") -fig = plt.figure(figsize=(20,10)) -ax1 = fig.add_subplot(1,2,1, xticks=[], yticks=[]) -ax1.imshow(imconvert(content)) -ax2 = fig.add_subplot(1,2,2, xticks=[], yticks=[]) -ax2.imshow(imconvert(style)) +if 0: + fig = plt.figure(figsize=(20,10)) + ax1 = fig.add_subplot(1,2,1, xticks=[], yticks=[]) + ax1.imshow(imconvert(content)) + ax2 = fig.add_subplot(1,2,2, xticks=[], yticks=[]) + ax2.imshow(imconvert(style)) #printing the vgg model vgg @@ -98,6 +104,9 @@ def gram_matrix(tensor): style_features = get_features(style, vgg) content_features = get_features(content, vgg) + +print("done get_features()") + style_grams = {layer: gram_matrix(style_features[layer]) for layer in style_features} #we could start with random image, but it would be good to start with content image @@ -118,7 +127,9 @@ def gram_matrix(tensor): optimizer = torch.optim.Adam([target], lr=0.003) steps = 2400 -print_every = 400 +print_every = 40 + +print("going to convert images") for i in range(1,steps+1): @@ -127,17 +138,11 @@ def gram_matrix(tensor): style_loss = 0 for layer in style_weights: - target_feature = target_features[layer] - _, d, h, w = target_feature.shape - target_gram = gram_matrix(target_feature) - style_gram = style_grams[layer] - layer_style_loss = style_weights[layer]*torch.mean((target_gram - style_gram)**2) - style_loss += layer_style_loss/ (d*h*w) total_loss = style_weight*style_loss + content_weight*content_loss @@ -146,21 +151,24 @@ def gram_matrix(tensor): total_loss.backward() optimizer.step() - if i%print_every==0: - print('Total Loss: ', total_loss.item()) + if i % print_every==0: + print(f'{i}: Total Loss: ', total_loss.item()) plt.imshow(imconvert(target)) - - -fig = plt.figure(figsize=(22,10)) -ax1 = fig.add_subplot(1,3,1, xticks=[], yticks=[]) -ax1.imshow(imconvert(content)) -ax2 = fig.add_subplot(1,3,2, xticks=[], yticks=[]) -ax2.imshow(imconvert(style)) - -ax3 = fig.add_subplot(1,3,3, xticks=[], yticks=[]) -ax3.imshow(imconvert(target)) - -converted_name = REPO_ROOT / "assets/surya2_style.jpg" -plt.imsave(str(converted_name), imconvert(target)) + dst_name = DST_DIR / f"{content_image.stem}_{i:05d}.jpg" + skimage.io.imsave(str(dst_name), imconvert(target)) + # pil_img = Image(imconvert(target)) + # pil_img.save(str(dst_name)) +if 0: + fig = plt.figure(figsize=(22,10)) + ax1 = fig.add_subplot(1,3,1, xticks=[], yticks=[]) + ax1.imshow(imconvert(content)) + ax2 = fig.add_subplot(1,3,2, xticks=[], yticks=[]) + ax2.imshow(imconvert(style)) + + ax3 = fig.add_subplot(1,3,3, xticks=[], yticks=[]) + ax3.imshow(imconvert(target)) + + converted_name = REPO_ROOT / "assets/surya2_style.jpg" + plt.imsave(str(converted_name), imconvert(target)) From 29ceee6936b3ab54558a99094f3ae5e5df15a006 Mon Sep 17 00:00:00 2001 From: katsunori waragai Date: Thu, 15 Jun 2023 18:22:26 +0900 Subject: [PATCH 06/11] under simplify --- style_transfer.py | 129 ++++++++++++++++++---------------------------- 1 file changed, 51 insertions(+), 78 deletions(-) diff --git a/style_transfer.py b/style_transfer.py index 360b2b6..8d1d4a3 100755 --- a/style_transfer.py +++ b/style_transfer.py @@ -68,107 +68,80 @@ def gram_matrix(tensor): return tensor -vgg = models.vgg19(pretrained=True).features +if __name__ == "__main__": + vgg = models.vgg19(pretrained=True).features -for param in vgg.parameters(): - param.requires_grad_(False) + for param in vgg.parameters(): + param.requires_grad_(False) -device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') -print('Cuda Available: ', torch.cuda.is_available()) -vgg.to(device) + device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') + print('Cuda Available: ', torch.cuda.is_available()) + vgg.to(device) -content_image = REPO_ROOT / "assets/surya2.jpg" -#load content image -content = load_image(str(content_image)).to(device) + content_image = REPO_ROOT / "assets/surya2.jpg" + #load content image + content = load_image(str(content_image)).to(device) -style_image = REPO_ROOT / "assets/oily_mcoilface.jpg" -#load style image -style = load_image(str(style_image), shape=content.shape[-2:]).to(device) + style_image = REPO_ROOT / "assets/oily_mcoilface.jpg" + #load style image + style = load_image(str(style_image), shape=content.shape[-2:]).to(device) -print("loaded images") + print("loaded images") -if 0: - fig = plt.figure(figsize=(20,10)) - ax1 = fig.add_subplot(1,2,1, xticks=[], yticks=[]) - ax1.imshow(imconvert(content)) - ax2 = fig.add_subplot(1,2,2, xticks=[], yticks=[]) - ax2.imshow(imconvert(style)) + style_features = get_features(style, vgg) + content_features = get_features(content, vgg) -#printing the vgg model -vgg + print("done get_features()") + style_grams = {layer: gram_matrix(style_features[layer]) for layer in style_features} -style_features = get_features(style, vgg) -content_features = get_features(content, vgg) + #we could start with random image, but it would be good to start with content image + target = content.clone().requires_grad_(True).to(device) -print("done get_features()") + style_weights = {'conv1_1': 1., + 'conv2_1': 0.8, + 'conv3_1': 0.5, + 'conv4_1': 0.3, + 'conv5_1': 0.1} -style_grams = {layer: gram_matrix(style_features[layer]) for layer in style_features} -#we could start with random image, but it would be good to start with content image -target = content.clone().requires_grad_(True).to(device) + content_weight = 1 # alpha + style_weight = 5e6 # beta -style_weights = {'conv1_1': 1., - 'conv2_1': 0.8, - 'conv3_1': 0.5, - 'conv4_1': 0.3, - 'conv5_1': 0.1} + optimizer = torch.optim.Adam([target], lr=0.003) + steps = 2400 + print_every = 40 -content_weight = 1 # alpha -style_weight = 5e6 # beta + print("going to convert images") + for i in range(1,steps+1): -optimizer = torch.optim.Adam([target], lr=0.003) + target_features = get_features(target, vgg) + content_loss = torch.mean((content_features['conv4_2']-target_features['conv4_2'])**2) -steps = 2400 -print_every = 40 + style_loss = 0 + for layer in style_weights: + target_feature = target_features[layer] + _, d, h, w = target_feature.shape + target_gram = gram_matrix(target_feature) + style_gram = style_grams[layer] + layer_style_loss = style_weights[layer]*torch.mean((target_gram - style_gram)**2) + style_loss += layer_style_loss/ (d*h*w) -print("going to convert images") - -for i in range(1,steps+1): - - target_features = get_features(target, vgg) - content_loss = torch.mean((content_features['conv4_2']-target_features['conv4_2'])**2) - - style_loss = 0 - for layer in style_weights: - target_feature = target_features[layer] - _, d, h, w = target_feature.shape - target_gram = gram_matrix(target_feature) - style_gram = style_grams[layer] - layer_style_loss = style_weights[layer]*torch.mean((target_gram - style_gram)**2) - style_loss += layer_style_loss/ (d*h*w) - - total_loss = style_weight*style_loss + content_weight*content_loss - - optimizer.zero_grad() - total_loss.backward() - optimizer.step() - - if i % print_every==0: - print(f'{i}: Total Loss: ', total_loss.item()) - plt.imshow(imconvert(target)) - dst_name = DST_DIR / f"{content_image.stem}_{i:05d}.jpg" - skimage.io.imsave(str(dst_name), imconvert(target)) - # pil_img = Image(imconvert(target)) - # pil_img.save(str(dst_name)) -if 0: - fig = plt.figure(figsize=(22,10)) - ax1 = fig.add_subplot(1,3,1, xticks=[], yticks=[]) - ax1.imshow(imconvert(content)) - ax2 = fig.add_subplot(1,3,2, xticks=[], yticks=[]) - ax2.imshow(imconvert(style)) - - ax3 = fig.add_subplot(1,3,3, xticks=[], yticks=[]) - ax3.imshow(imconvert(target)) - - converted_name = REPO_ROOT / "assets/surya2_style.jpg" - plt.imsave(str(converted_name), imconvert(target)) + total_loss = style_weight*style_loss + content_weight*content_loss + optimizer.zero_grad() + total_loss.backward() + optimizer.step() + if i % print_every==0: + print(f'{i}: Total Loss: ', total_loss.item()) + plt.imshow(imconvert(target)) + dst_name = DST_DIR / f"{content_image.stem}_{i:05d}.jpg" + skimage.io.imsave(str(dst_name), imconvert(target)) From 9e218982a62e2ceb11e224d42931b2292d28e2db Mon Sep 17 00:00:00 2001 From: katsunori waragai Date: Thu, 15 Jun 2023 18:34:47 +0900 Subject: [PATCH 07/11] under simplify --- style_transfer.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/style_transfer.py b/style_transfer.py index 8d1d4a3..1ba9948 100755 --- a/style_transfer.py +++ b/style_transfer.py @@ -68,24 +68,19 @@ def gram_matrix(tensor): return tensor -if __name__ == "__main__": +def style_convert(style_image: Path, content_image: Path): vgg = models.vgg19(pretrained=True).features for param in vgg.parameters(): param.requires_grad_(False) - device = torch.device('cuda' if torch.cuda.is_available() else 'cpu') print('Cuda Available: ', torch.cuda.is_available()) vgg.to(device) - - - content_image = REPO_ROOT / "assets/surya2.jpg" #load content image content = load_image(str(content_image)).to(device) - style_image = REPO_ROOT / "assets/oily_mcoilface.jpg" #load style image style = load_image(str(style_image), shape=content.shape[-2:]).to(device) @@ -115,7 +110,7 @@ def gram_matrix(tensor): optimizer = torch.optim.Adam([target], lr=0.003) - steps = 2400 + steps = 200 print_every = 40 print("going to convert images") @@ -145,3 +140,16 @@ def gram_matrix(tensor): plt.imshow(imconvert(target)) dst_name = DST_DIR / f"{content_image.stem}_{i:05d}.jpg" skimage.io.imsave(str(dst_name), imconvert(target)) + +if __name__ == "__main__": + import argparse + parser = argparse.ArgumentParser("style transfer") + parser.add_argument("style", help="style image") + parser.add_argument("content", help="image to style transfer") + args = parser.parse_args() + + style_image = Path(args.style) + content_image = Path(args.content) + + style_convert(style_image, content_image) + From d435b47e02a36a55fd1c50616264c5c634de84ed Mon Sep 17 00:00:00 2001 From: katsunori waragai Date: Thu, 15 Jun 2023 18:53:58 +0900 Subject: [PATCH 08/11] keep image size --- style_transfer.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/style_transfer.py b/style_transfer.py index 1ba9948..4f0a3e7 100755 --- a/style_transfer.py +++ b/style_transfer.py @@ -3,9 +3,8 @@ from pathlib import Path import numpy as np -import matplotlib.pyplot as plt from PIL import Image -# import cv2 +import cv2 import skimage import torch @@ -137,9 +136,11 @@ def style_convert(style_image: Path, content_image: Path): if i % print_every==0: print(f'{i}: Total Loss: ', total_loss.item()) - plt.imshow(imconvert(target)) dst_name = DST_DIR / f"{content_image.stem}_{i:05d}.jpg" - skimage.io.imsave(str(dst_name), imconvert(target)) + converted = imconvert(target) + [h, w] = content.shape[:2] + resized = cv2.resize(converted, (w, h), interpolation=cv2.INTER_LINEAR) + skimage.io.imsave(str(dst_name), resized) if __name__ == "__main__": import argparse From df7c79a3c9db96dfc57d02eb43f73a52b0a8e21d Mon Sep 17 00:00:00 2001 From: katsunori waragai Date: Thu, 15 Jun 2023 19:14:56 +0900 Subject: [PATCH 09/11] fixed image size --- style_transfer.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/style_transfer.py b/style_transfer.py index 4f0a3e7..99312f1 100755 --- a/style_transfer.py +++ b/style_transfer.py @@ -79,6 +79,7 @@ def style_convert(style_image: Path, content_image: Path): #load content image content = load_image(str(content_image)).to(device) + [height, width] = cv2.imread(str(content_image)).shape[:2] #load style image style = load_image(str(style_image), shape=content.shape[-2:]).to(device) @@ -138,8 +139,7 @@ def style_convert(style_image: Path, content_image: Path): print(f'{i}: Total Loss: ', total_loss.item()) dst_name = DST_DIR / f"{content_image.stem}_{i:05d}.jpg" converted = imconvert(target) - [h, w] = content.shape[:2] - resized = cv2.resize(converted, (w, h), interpolation=cv2.INTER_LINEAR) + resized = cv2.resize(converted, (width, height), interpolation=cv2.INTER_LINEAR) skimage.io.imsave(str(dst_name), resized) if __name__ == "__main__": @@ -153,4 +153,3 @@ def style_convert(style_image: Path, content_image: Path): content_image = Path(args.content) style_convert(style_image, content_image) - From 8b12dd09504090ab7f7440b5cde4c52e68f1eb75 Mon Sep 17 00:00:00 2001 From: katsunori waragai Date: Thu, 15 Jun 2023 19:32:16 +0900 Subject: [PATCH 10/11] strange color --- style_transfer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/style_transfer.py b/style_transfer.py index 99312f1..5323b28 100755 --- a/style_transfer.py +++ b/style_transfer.py @@ -137,8 +137,10 @@ def style_convert(style_image: Path, content_image: Path): if i % print_every==0: print(f'{i}: Total Loss: ', total_loss.item()) - dst_name = DST_DIR / f"{content_image.stem}_{i:05d}.jpg" + dst_name0 = DST_DIR / f"{content_image.stem}_{i:05d}.jpg" + dst_name = DST_DIR / f"{content_image.stem}_{i:05d}_original_size.jpg" converted = imconvert(target) + skimage.io.imsave(str(dst_name0), converted) resized = cv2.resize(converted, (width, height), interpolation=cv2.INTER_LINEAR) skimage.io.imsave(str(dst_name), resized) From 72397d1ac4a494598b469ee3f6f3ee6a94e62ba8 Mon Sep 17 00:00:00 2001 From: katsunori waragai Date: Thu, 15 Jun 2023 20:50:44 +0900 Subject: [PATCH 11/11] added to requirements.txt --- requirements.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/requirements.txt b/requirements.txt index 82e4df1..1644d8c 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,6 @@ +numpy +opencv-python +scikit-image torch torchvision Pillow==4.1.1