CoCalc Logo Icon
StoreFeaturesDocsShareSupportNewsAboutSign UpSign In
amanchadha

Real-time collaboration for Jupyter Notebooks, Linux Terminals, LaTeX, VS Code, R IDE, and more,
all in one place. Commercial Alternative to JupyterHub.

GitHub Repository: amanchadha/coursera-deep-learning-specialization
 Path: blob/master/C4 - Convolutional Neural Networks/Week 4/Neural Style Transfer/nst_utils.py
Views: 4818
1
### Part of this code is due to the MatConvNet team and is used to load the parameters of the pretrained VGG19 model in the notebook ###

2


3
import os

4
import sys

5
import scipy.io

6
import scipy.misc

7
import matplotlib.pyplot as plt

8
from matplotlib.pyplot import imshow

9
from PIL import Image

10
from nst_utils import *

11


12
import numpy as np

13
import tensorflow as tf

14


15
class CONFIG:

16
    IMAGE_WIDTH = 400

17
    IMAGE_HEIGHT = 300

18
    COLOR_CHANNELS = 3

19
    NOISE_RATIO = 0.6

20
    MEANS = np.array([123.68, 116.779, 103.939]).reshape((1,1,1,3)) 

21
    VGG_MODEL = 'pretrained-model/imagenet-vgg-verydeep-19.mat' # Pick the VGG 19-layer model by from the paper "Very Deep Convolutional Networks for Large-Scale Image Recognition".

22
    STYLE_IMAGE = 'images/stone_style.jpg' # Style image to use.

23
    CONTENT_IMAGE = 'images/content300.jpg' # Content image to use.

24
    OUTPUT_DIR = 'output/'

25
    

26
def load_vgg_model(path):

27
    """

28
    Returns a model for the purpose of 'painting' the picture.

29
    Takes only the convolution layer weights and wrap using the TensorFlow

30
    Conv2d, Relu and AveragePooling layer. VGG actually uses maxpool but

31
    the paper indicates that using AveragePooling yields better results.

32
    The last few fully connected layers are not used.

33
    Here is the detailed configuration of the VGG model:

34
        0 is conv1_1 (3, 3, 3, 64)

35
        1 is relu

36
        2 is conv1_2 (3, 3, 64, 64)

37
        3 is relu    

38
        4 is maxpool

39
        5 is conv2_1 (3, 3, 64, 128)

40
        6 is relu

41
        7 is conv2_2 (3, 3, 128, 128)

42
        8 is relu

43
        9 is maxpool

44
        10 is conv3_1 (3, 3, 128, 256)

45
        11 is relu

46
        12 is conv3_2 (3, 3, 256, 256)

47
        13 is relu

48
        14 is conv3_3 (3, 3, 256, 256)

49
        15 is relu

50
        16 is conv3_4 (3, 3, 256, 256)

51
        17 is relu

52
        18 is maxpool

53
        19 is conv4_1 (3, 3, 256, 512)

54
        20 is relu

55
        21 is conv4_2 (3, 3, 512, 512)

56
        22 is relu

57
        23 is conv4_3 (3, 3, 512, 512)

58
        24 is relu

59
        25 is conv4_4 (3, 3, 512, 512)

60
        26 is relu

61
        27 is maxpool

62
        28 is conv5_1 (3, 3, 512, 512)

63
        29 is relu

64
        30 is conv5_2 (3, 3, 512, 512)

65
        31 is relu

66
        32 is conv5_3 (3, 3, 512, 512)

67
        33 is relu

68
        34 is conv5_4 (3, 3, 512, 512)

69
        35 is relu

70
        36 is maxpool

71
        37 is fullyconnected (7, 7, 512, 4096)

72
        38 is relu

73
        39 is fullyconnected (1, 1, 4096, 4096)

74
        40 is relu

75
        41 is fullyconnected (1, 1, 4096, 1000)

76
        42 is softmax

77
    """

78
    

79
    vgg = scipy.io.loadmat(path)

80


81
    vgg_layers = vgg['layers']

82
    

83
    def _weights(layer, expected_layer_name):

84
        """

85
        Return the weights and bias from the VGG model for a given layer.

86
        """

87
        wb = vgg_layers[0][layer][0][0][2]

88
        W = wb[0][0]

89
        b = wb[0][1]

90
        layer_name = vgg_layers[0][layer][0][0][0][0]

91
        assert layer_name == expected_layer_name

92
        return W, b

93


94
        return W, b

95


96
    def _relu(conv2d_layer):

97
        """

98
        Return the RELU function wrapped over a TensorFlow layer. Expects a

99
        Conv2d layer input.

100
        """

101
        return tf.nn.relu(conv2d_layer)

102


103
    def _conv2d(prev_layer, layer, layer_name):

104
        """

105
        Return the Conv2D layer using the weights, biases from the VGG

106
        model at 'layer'.

107
        """

108
        W, b = _weights(layer, layer_name)

109
        W = tf.constant(W)

110
        b = tf.constant(np.reshape(b, (b.size)))

111
        return tf.nn.conv2d(prev_layer, filter=W, strides=[1, 1, 1, 1], padding='SAME') + b

112


113
    def _conv2d_relu(prev_layer, layer, layer_name):

114
        """

115
        Return the Conv2D + RELU layer using the weights, biases from the VGG

116
        model at 'layer'.

117
        """

118
        return _relu(_conv2d(prev_layer, layer, layer_name))

119


120
    def _avgpool(prev_layer):

121
        """

122
        Return the AveragePooling layer.

123
        """

124
        return tf.nn.avg_pool(prev_layer, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')

125


126
    # Constructs the graph model.

127
    graph = {}

128
    graph['input']   = tf.Variable(np.zeros((1, CONFIG.IMAGE_HEIGHT, CONFIG.IMAGE_WIDTH, CONFIG.COLOR_CHANNELS)), dtype = 'float32')

129
    graph['conv1_1']  = _conv2d_relu(graph['input'], 0, 'conv1_1')

130
    graph['conv1_2']  = _conv2d_relu(graph['conv1_1'], 2, 'conv1_2')

131
    graph['avgpool1'] = _avgpool(graph['conv1_2'])

132
    graph['conv2_1']  = _conv2d_relu(graph['avgpool1'], 5, 'conv2_1')

133
    graph['conv2_2']  = _conv2d_relu(graph['conv2_1'], 7, 'conv2_2')

134
    graph['avgpool2'] = _avgpool(graph['conv2_2'])

135
    graph['conv3_1']  = _conv2d_relu(graph['avgpool2'], 10, 'conv3_1')

136
    graph['conv3_2']  = _conv2d_relu(graph['conv3_1'], 12, 'conv3_2')

137
    graph['conv3_3']  = _conv2d_relu(graph['conv3_2'], 14, 'conv3_3')

138
    graph['conv3_4']  = _conv2d_relu(graph['conv3_3'], 16, 'conv3_4')

139
    graph['avgpool3'] = _avgpool(graph['conv3_4'])

140
    graph['conv4_1']  = _conv2d_relu(graph['avgpool3'], 19, 'conv4_1')

141
    graph['conv4_2']  = _conv2d_relu(graph['conv4_1'], 21, 'conv4_2')

142
    graph['conv4_3']  = _conv2d_relu(graph['conv4_2'], 23, 'conv4_3')

143
    graph['conv4_4']  = _conv2d_relu(graph['conv4_3'], 25, 'conv4_4')

144
    graph['avgpool4'] = _avgpool(graph['conv4_4'])

145
    graph['conv5_1']  = _conv2d_relu(graph['avgpool4'], 28, 'conv5_1')

146
    graph['conv5_2']  = _conv2d_relu(graph['conv5_1'], 30, 'conv5_2')

147
    graph['conv5_3']  = _conv2d_relu(graph['conv5_2'], 32, 'conv5_3')

148
    graph['conv5_4']  = _conv2d_relu(graph['conv5_3'], 34, 'conv5_4')

149
    graph['avgpool5'] = _avgpool(graph['conv5_4'])

150
    

151
    return graph

152


153
def generate_noise_image(content_image, noise_ratio = CONFIG.NOISE_RATIO):

154
    """

155
    Generates a noisy image by adding random noise to the content_image

156
    """

157
    

158
    # Generate a random noise_image

159
    noise_image = np.random.uniform(-20, 20, (1, CONFIG.IMAGE_HEIGHT, CONFIG.IMAGE_WIDTH, CONFIG.COLOR_CHANNELS)).astype('float32')

160
    

161
    # Set the input_image to be a weighted average of the content_image and a noise_image

162
    input_image = noise_image * noise_ratio + content_image * (1 - noise_ratio)

163
    

164
    return input_image

165


166


167
def reshape_and_normalize_image(image):

168
    """

169
    Reshape and normalize the input image (content or style)

170
    """

171
    

172
    # Reshape image to mach expected input of VGG16

173
    image = np.reshape(image, ((1,) + image.shape))

174
    

175
    # Substract the mean to match the expected input of VGG16

176
    image = image - CONFIG.MEANS

177
    

178
    return image

179


180


181
def save_image(path, image):

182
    

183
    # Un-normalize the image so that it looks good

184
    image = image + CONFIG.MEANS

185
    

186
    # Clip and Save the image

187
    image = np.clip(image[0], 0, 255).astype('uint8')

188
    scipy.misc.imsave(path, image)

189