533d585
user_wwq 3 years ago
49 changed file(s) with 1956 addition(s) and 17 deletion(s). Raw diff Collapse all Expand all
+49
-7
_README.ipynb less more
1616 " - ```_README.md```*-----说明文档*\n",
1717 " - ```app_spec.yml```*-----定义项目的输入输出，为部署服务*\n",
1818 " - ```coding_here.ipynb```*-----输入并运行代码*"
19 ]
20 },
21 {
22 "cell_type": "code",
23 "execution_count": null,
24 "metadata": {},
25 "outputs": [],
26 "source": [
27 "def handle(conf):\n",
28 " \"\"\"\n",
29 " 该方法是部署之后，其他人调用你的服务时候的处理方法。\n",
30 " 请按规范填写参数结构，这样我们就能替你自动生成配置文件，方便其他人的调用。\n",
31 " 范例：\n",
32 " params['key'] = value # value_type: str # description: some description\n",
33 " value_type 可以选择：img, video, audio, str, int, float, [int], [str], [float]\n",
34 " 参数请放到params字典中，我们会自动解析该变量。\n",
35 " \"\"\"\n",
36 "\n",
37 " param1 = conf['param1'] # value_type: str # description: some description\n",
38 " # add your code\n",
39 " return {'ret1': 'cat'}\n",
40 " "
41 ]
42 },
43 {
44 "cell_type": "code",
45 "execution_count": 1,
46 "metadata": {},
47 "outputs": [
48 {
49 "data": {
50 "text/plain": [
51 "'/home/jovyan/work'"
52 ]
53 },
54 "execution_count": 1,
55 "metadata": {},
56 "output_type": "execute_result"
57 }
58 ],
59 "source": [
60 "pwd"
1961 ]
2062 },
2163 {
120162 ],
121163 "metadata": {
122164 "kernelspec": {
123 "display_name": "Python 3",
165 "display_name": "Python 3 (ipykernel)",
124166 "language": "python",
125167 "name": "python3"
126168 },
134176 "name": "python",
135177 "nbconvert_exporter": "python",
136178 "pygments_lexer": "ipython3",
137 "version": "3.5.2"
138 },
179 "version": "3.7.5"
180 },
139181 "pycharm": {
140182 "stem_cell": {
141183 "cell_type": "raw",
142 "source": [],
143184 "metadata": {
144185 "collapsed": false
145 }
146 }
147 }
186 },
187 "source": []
188 }
189 }
148190 },
149191 "nbformat": 4,
150192 "nbformat_minor": 2
+10
-0
app_spec.yml less more
0 input:
1 Photo:
2 name: Photo
3 value_type: img
4 description: 请传入128*128的图片
5 output:
6 Output:
7 name: Output
8 value_type: img
9 description: ''
+10
-0
etc/.ipynb_checkpoints/dev-indices-checkpoint.txt less more
0 Indices (0-indexed) of the 100 images held out from training.
1 [ 113 509 242 280 533 638 644 698 751 832 10989 16008
2 13473 6659 20401 24841 26378 8103 11730 8363 16512 6736 27666 30287
3 6685 30696 16591 8424 26689 21078 27971 7202 6615 36150 9681 13137
4 1598 9726 4825 2864 1346 21784 4159 13270 19239 9844 16056 2822
5 15792 19837 5198 19980 30042 36491 15648 20315 3604 8020 1108 18235
6 16373 25717 32200 10547 6786 31384 33999 25763 20226 9447 4573 5938
7 1837 25121 17611 32751 28158 29381 13090 32210 17027 30171 12001 16240
8 22205 11808 20113 10682 33338 24015 15154 10449 11373 8736 26320 4095
9 13855 23504 2004 33307]
etc/.ipynb_checkpoints/outpainting-checkpoint.png less more
Binary diff not shown
etc/.ipynb_checkpoints/recursive-checkpoint.png less more
Binary diff not shown
etc/.ipynb_checkpoints/results-checkpoint.png less more
Binary diff not shown
+45
-0
handler.py less more
0 import tensorflow.compat.v1 as tf
1 tf.disable_v2_behavior()
2 tf.reset_default_graph()
3 import numpy as np
4 from PIL import Image
5 import model
6 import util
7 import os
8 import sys
9
10 model_PATH='/home/jovyan/work/src/output/models/model2000.ckpt'
11
12 def load_demo_image(in_PATH):
13 img = np.array(Image.open(in_PATH).convert('RGB'))[np.newaxis] / 255.0
14 img_p = util.preprocess_images_outpainting(img)
15 return img_p
16
17 def inference(model_PATH, img_p):
18 G_Z = tf.placeholder(tf.float32, shape=[None, IMAGE_SZ, IMAGE_SZ, 4], name='G_Z')
19 G_sample = model.generator(G_Z)
20
21 saver = tf.train.Saver()
22 with tf.Session() as sess:
23 saver.restore(sess, model_PATH)
24 output, = sess.run([G_sample], feed_dict={G_Z: img_p})
25 img_norm = (output[0] * 255.0).astype(np.uint8)
26 img = Image.fromarray(img_norm, 'RGB')
27 #util.save_image(output[0], out_PATH)
28 return img
29
30 def handle(conf):
31 """
32 该方法是部署之后，其他人调用你的服务时候的处理方法。
33 请按规范填写参数结构，这样我们就能替你自动生成配置文件，方便其他人的调用。
34 范例：
35 params['key'] = value # value_type: str # description: some description
36 value_type 可以选择：img, video, audio, str, int, float, [int], [str], [float]
37 参数请放到params字典中，我们会自动解析该变量。
38 """
39 base64_str = conf['Photo']
40 image = load_demo_image(base64_str, image_size, device)
41 res = inference(model, image)
42 # add your code
43 return {'Output': res}
44 ⏎
images/.ipynb_checkpoints/city_128-checkpoint.png less more
Binary diff not shown
images/test.png less more
Binary diff not shown
+10
-0
job_logs/job-gpu-62b5d2d8c06b81cd38279610.log less more
0 2022-06-24T15:06:06.543501249Z SYSTEM: Preparing env...
1 2022-06-24T15:06:07.124450272Z SYSTEM: Running...
2 2022-06-24T15:06:07.897430232Z 2022-06-24 23:06:07.895520: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
3 2022-06-24T15:06:09.566498086Z Imported model (for Places365, 128x128 images)
4 2022-06-24T15:06:12.24384624Z Traceback (most recent call last):
5 2022-06-24T15:06:12.243895488Z File "src/train.py", line 15, in <module>
6 2022-06-24T15:06:12.263438511Z tf.reset_default_graph()
7 2022-06-24T15:06:12.263468552Z AttributeError: module 'tensorflow' has no attribute 'reset_default_graph'
8 2022-06-24T15:06:12.683968769Z SYSTEM: Finishing...
9 2022-06-24T15:06:12.897833055Z SYSTEM: Error Exists!
+10
-0
job_logs/job-gpu-62b5d344b5c4eec184cc05b0.log less more
0 2022-06-24T15:07:54.304456417Z SYSTEM: Preparing env...
1 2022-06-24T15:07:54.807933369Z SYSTEM: Running...
2 2022-06-24T15:07:55.517547472Z 2022-06-24 23:07:55.513238: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
3 2022-06-24T15:07:57.101646267Z Imported model (for Places365, 128x128 images)
4 2022-06-24T15:07:59.609309108Z Traceback (most recent call last):
5 2022-06-24T15:07:59.609359562Z File "src/train.py", line 64, in <module>
6 2022-06-24T15:07:59.614462932Z G_Z = tf.placeholder(tf.float32, shape=[None, IMAGE_SZ, IMAGE_SZ, 4], name='G_Z')
7 2022-06-24T15:07:59.614484248Z AttributeError: module 'tensorflow' has no attribute 'placeholder'
8 2022-06-24T15:08:00.047025825Z SYSTEM: Finishing...
9 2022-06-24T15:08:00.254692017Z SYSTEM: Error Exists!
+12
-0
job_logs/job-gpu-62b5d39fd4e7f8c811b53eb1.log less more
0 2022-06-24T15:09:25.204980934Z SYSTEM: Preparing env...
1 2022-06-24T15:09:25.719685215Z SYSTEM: Running...
2 2022-06-24T15:09:26.429518202Z 2022-06-24 23:09:26.428748: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
3 2022-06-24T15:09:28.084842936Z Imported model (for Places365, 128x128 images)
4 2022-06-24T15:09:30.663258107Z Traceback (most recent call last):
5 2022-06-24T15:09:30.663303708Z File "src/train.py", line 64, in <module>
6 2022-06-24T15:09:30.663549466Z G_Z = tf.placeholder(tf.float32, shape=[None, IMAGE_SZ, IMAGE_SZ, 4], name='G_Z')
7 2022-06-24T15:09:30.664301035Z File "/home/jovyan/.virtualenvs/basenv/lib/python3.7/site-packages/tensorflow/python/ops/array_ops.py", line 3097, in placeholder
8 2022-06-24T15:09:30.687425674Z raise RuntimeError("tf.placeholder() is not compatible with "
9 2022-06-24T15:09:30.687454335Z RuntimeError: tf.placeholder() is not compatible with eager execution.
10 2022-06-24T15:09:31.089543597Z SYSTEM: Finishing...
11 2022-06-24T15:09:31.296093256Z SYSTEM: Error Exists!
+10
-0
job_logs/job-gpu-62b5d4052a85ae797c345e17.log less more
0 2022-06-24T15:11:08.102681479Z SYSTEM: Preparing env...
1 2022-06-24T15:11:08.698452928Z SYSTEM: Running...
2 2022-06-24T15:11:09.445497917Z 2022-06-24 23:11:09.437930: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
3 2022-06-24T15:11:11.078881179Z Imported model (for Places365, 128x128 images)
4 2022-06-24T15:11:13.662043015Z Traceback (most recent call last):
5 2022-06-24T15:11:13.662088455Z File "src/train.py", line 64, in <module>
6 2022-06-24T15:11:13.667490013Z reset_graph()
7 2022-06-24T15:11:13.667534099Z NameError: name 'reset_graph' is not defined
8 2022-06-24T15:11:14.148404869Z SYSTEM: Finishing...
9 2022-06-24T15:11:14.346444169Z SYSTEM: Error Exists!
+12
-0
job_logs/job-gpu-62b5d43bc06b81cd38279613.log less more
0 2022-06-24T15:12:01.617089532Z SYSTEM: Preparing env...
1 2022-06-24T15:12:02.203109161Z SYSTEM: Running...
2 2022-06-24T15:12:03.004629571Z 2022-06-24 23:12:03.003884: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
3 2022-06-24T15:12:04.639278285Z Imported model (for Places365, 128x128 images)
4 2022-06-24T15:12:07.282805014Z Traceback (most recent call last):
5 2022-06-24T15:12:07.282856749Z File "src/train.py", line 71, in <module>
6 2022-06-24T15:12:07.283554986Z data = np.load('places/places_128.npz')
7 2022-06-24T15:12:07.283591663Z File "/usr/local/lib/python3.7/dist-packages/numpy/lib/npyio.py", line 428, in load
8 2022-06-24T15:12:07.299470153Z fid = open(os_fspath(file), "rb")
9 2022-06-24T15:12:07.299489677Z FileNotFoundError: [Errno 2] No such file or directory: 'places/places_128.npz'
10 2022-06-24T15:12:07.850426625Z SYSTEM: Finishing...
11 2022-06-24T15:12:08.075164736Z SYSTEM: Error Exists!
+9
-0
job_logs/job-gpu-62b94f70c584fdf74ee42dd3.log less more
0 2022-06-27T06:34:35.306224872Z SYSTEM: Preparing env...
1 2022-06-27T06:34:35.980785521Z SYSTEM: Running...
2 2022-06-27T06:34:36.741505339Z 2022-06-27 14:34:36.738257: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
3 2022-06-27T06:34:38.457388991Z WARNING:tensorflow:From /home/jovyan/.virtualenvs/basenv/lib/python3.7/site-packages/tensorflow/python/compat/v2_compat.py:96: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version.
4 2022-06-27T06:34:38.457442694Z Instructions for updating:
5 2022-06-27T06:34:38.457449025Z non-resource variables are not supported in the long term
6 2022-06-27T06:34:38.468014128Z Imported model (for Places365, 128x128 images)
7 2022-06-27T06:34:43.13214278Z Usage: python test.py [model_PATH] [in_PATH] [out_PATH]
8 2022-06-27T06:34:43.648430148Z SYSTEM: Finishing...
+14
-0
job_logs/job-gpu-62b94f9cb17f87f3a6d7445b.log less more
0 2022-06-27T06:35:15.051483054Z SYSTEM: Preparing env...
1 2022-06-27T06:35:15.65442522Z SYSTEM: Running...
2 2022-06-27T06:35:16.362517265Z 2022-06-27 14:35:16.361256: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
3 2022-06-27T06:35:18.024936822Z WARNING:tensorflow:From /home/jovyan/.virtualenvs/basenv/lib/python3.7/site-packages/tensorflow/python/compat/v2_compat.py:96: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version.
4 2022-06-27T06:35:18.024986723Z Instructions for updating:
5 2022-06-27T06:35:18.024991459Z non-resource variables are not supported in the long term
6 2022-06-27T06:35:18.029401275Z Imported model (for Places365, 128x128 images)
7 2022-06-27T06:35:20.683442548Z Traceback (most recent call last):
8 2022-06-27T06:35:20.683486794Z File "src/test.py", line 28, in <module>
9 2022-06-27T06:35:20.68361951Z G_sample = model.generator(G_Z)
10 2022-06-27T06:35:20.683632773Z File "/home/jovyan/work/src/model.py", line 12, in generator
11 2022-06-27T06:35:20.69064768Z with tf.variable_scope('G', reuse=tf.AUTO_REUSE):
12 2022-06-27T06:35:20.690694567Z AttributeError: module 'tensorflow' has no attribute 'variable_scope'
13 2022-06-27T06:35:21.152841351Z SYSTEM: Finishing...
+128
-0
job_logs/job-gpu-62b94fd2f752e3e25d1d3e30.log less more
0 2022-06-27T06:36:09.110278599Z SYSTEM: Preparing env...
1 2022-06-27T06:36:09.755697526Z SYSTEM: Running...
2 2022-06-27T06:36:10.582438824Z 2022-06-27 14:36:10.578599: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
3 2022-06-27T06:36:12.27114199Z WARNING:tensorflow:From /home/jovyan/.virtualenvs/basenv/lib/python3.7/site-packages/tensorflow/python/compat/v2_compat.py:96: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version.
4 2022-06-27T06:36:12.27118886Z Instructions for updating:
5 2022-06-27T06:36:12.271196741Z non-resource variables are not supported in the long term
6 2022-06-27T06:36:12.27588787Z Imported model (for Places365, 128x128 images)
7 2022-06-27T06:36:15.127571182Z WARNING:tensorflow:From /home/jovyan/work/src/model.py:20: conv2d (from tensorflow.python.keras.legacy_tf_layers.convolutional) is deprecated and will be removed in a future version.
8 2022-06-27T06:36:15.127625625Z Instructions for updating:
9 2022-06-27T06:36:15.127635424Z Use `tf.keras.layers.Conv2D` instead.
10 2022-06-27T06:36:15.127641681Z WARNING:tensorflow:From /home/jovyan/.virtualenvs/basenv/lib/python3.7/site-packages/tensorflow/python/keras/legacy_tf_layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer_v1) is deprecated and will be removed in a future version.
11 2022-06-27T06:36:15.127654313Z Instructions for updating:
12 2022-06-27T06:36:15.127659052Z Please use `layer.__call__` method instead.
13 2022-06-27T06:36:15.256535403Z WARNING:tensorflow:From /home/jovyan/work/src/model.py:79: conv2d_transpose (from tensorflow.python.keras.legacy_tf_layers.convolutional) is deprecated and will be removed in a future version.
14 2022-06-27T06:36:15.256581182Z Instructions for updating:
15 2022-06-27T06:36:15.256588253Z Use `tf.keras.layers.Conv2DTranspose` instead.
16 2022-06-27T06:36:15.327579181Z 2022-06-27 14:36:15.326484: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcuda.so.1
17 2022-06-27T06:36:15.413601745Z 2022-06-27 14:36:15.410378: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
18 2022-06-27T06:36:15.413646851Z 2022-06-27 14:36:15.411277: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1716] Found device 0 with properties:
19 2022-06-27T06:36:15.413654172Z pciBusID: 0000:00:08.0 name: Tesla P100-PCIE-16GB computeCapability: 6.0
20 2022-06-27T06:36:15.413657624Z coreClock: 1.3285GHz coreCount: 56 deviceMemorySize: 7.75GiB deviceMemoryBandwidth: 681.88GiB/s
21 2022-06-27T06:36:15.413660889Z 2022-06-27 14:36:15.411345: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
22 2022-06-27T06:36:15.424689012Z 2022-06-27 14:36:15.423389: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcublas.so.10
23 2022-06-27T06:36:15.432079412Z 2022-06-27 14:36:15.431364: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcufft.so.10
24 2022-06-27T06:36:15.496414752Z 2022-06-27 14:36:15.493736: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcurand.so.10
25 2022-06-27T06:36:15.593235058Z 2022-06-27 14:36:15.591720: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusolver.so.10
26 2022-06-27T06:36:15.599321627Z 2022-06-27 14:36:15.596773: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusparse.so.10
27 2022-06-27T06:36:15.805424915Z 2022-06-27 14:36:15.801512: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudnn.so.7
28 2022-06-27T06:36:15.805479188Z 2022-06-27 14:36:15.801981: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
29 2022-06-27T06:36:15.805486999Z 2022-06-27 14:36:15.803112: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
30 2022-06-27T06:36:15.805492776Z 2022-06-27 14:36:15.804002: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1858] Adding visible gpu devices: 0
31 2022-06-27T06:36:15.805497988Z 2022-06-27 14:36:15.804847: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN)to use the following CPU instructions in performance-critical operations: AVX2 FMA
32 2022-06-27T06:36:15.805503237Z To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
33 2022-06-27T06:36:15.81417475Z 2022-06-27 14:36:15.813899: I tensorflow/core/platform/profile_utils/cpu_utils.cc:104] CPU Frequency: 2499990000 Hz
34 2022-06-27T06:36:15.814946288Z 2022-06-27 14:36:15.814691: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x5547d70 initialized for platform Host (this does not guarantee that XLA will be used). Devices:
35 2022-06-27T06:36:15.814965987Z 2022-06-27 14:36:15.814721: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version
36 2022-06-27T06:36:15.978500134Z 2022-06-27 14:36:15.972620: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
37 2022-06-27T06:36:15.978540247Z 2022-06-27 14:36:15.973668: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x4916460 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:
38 2022-06-27T06:36:15.978547933Z 2022-06-27 14:36:15.973703: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla P100-PCIE-16GB, Compute Capability 6.0
39 2022-06-27T06:36:15.978553148Z 2022-06-27 14:36:15.974073: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
40 2022-06-27T06:36:15.978558435Z 2022-06-27 14:36:15.974960: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1716] Found device 0 with properties:
41 2022-06-27T06:36:15.978579396Z pciBusID: 0000:00:08.0 name: Tesla P100-PCIE-16GB computeCapability: 6.0
42 2022-06-27T06:36:15.978583413Z coreClock: 1.3285GHz coreCount: 56 deviceMemorySize: 7.75GiB deviceMemoryBandwidth: 681.88GiB/s
43 2022-06-27T06:36:15.978586337Z 2022-06-27 14:36:15.974996: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
44 2022-06-27T06:36:15.978589107Z 2022-06-27 14:36:15.975027: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcublas.so.10
45 2022-06-27T06:36:15.97859187Z 2022-06-27 14:36:15.975052: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcufft.so.10
46 2022-06-27T06:36:15.978594744Z 2022-06-27 14:36:15.975075: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcurand.so.10
47 2022-06-27T06:36:15.978597775Z 2022-06-27 14:36:15.975098: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusolver.so.10
48 2022-06-27T06:36:15.978600631Z 2022-06-27 14:36:15.975121: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusparse.so.10
49 2022-06-27T06:36:15.978604493Z 2022-06-27 14:36:15.975144: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudnn.so.7
50 2022-06-27T06:36:15.978607328Z 2022-06-27 14:36:15.975351: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
51 2022-06-27T06:36:15.978610359Z 2022-06-27 14:36:15.976383: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
52 2022-06-27T06:36:15.978613279Z 2022-06-27 14:36:15.977323: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1858] Adding visible gpu devices: 0
53 2022-06-27T06:36:15.978616007Z 2022-06-27 14:36:15.977373: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
54 2022-06-27T06:36:16.61051414Z 2022-06-27 14:36:16.609849: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1257] Device interconnect StreamExecutor with strength 1 edge matrix:
55 2022-06-27T06:36:16.610556469Z 2022-06-27 14:36:16.609912: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1263] 0
56 2022-06-27T06:36:16.61056136Z 2022-06-27 14:36:16.609925: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1276] 0: N
57 2022-06-27T06:36:16.621638578Z 2022-06-27 14:36:16.618766: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
58 2022-06-27T06:36:16.621703314Z 2022-06-27 14:36:16.619824: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
59 2022-06-27T06:36:16.621711487Z 2022-06-27 14:36:16.620774: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1402] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 7116 MB memory) -> physical GPU (device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:08.0, compute capability: 6.0)
60 2022-06-27T06:36:16.659863984Z 2022-06-27 14:36:16.659585: W tensorflow/core/util/tensor_slice_reader.cc:95] Could not open /home/jovyan/work/src/output/models: Failed precondition: /home/jovyan/work/src/output/models; Is a directory: perhaps your file is in a different file format and you need to use a different restore operator?
61 2022-06-27T06:36:16.661458563Z 2022-06-27 14:36:16.661189: W tensorflow/core/util/tensor_slice_reader.cc:95] Could not open /home/jovyan/work/src/output/models: Failed precondition: /home/jovyan/work/src/output/models; Is a directory: perhaps your file is in a different file format and you need to use a different restore operator?
62 2022-06-27T06:36:16.66147741Z 2022-06-27 14:36:16.661256: W tensorflow/core/framework/op_kernel.cc:1767] OP_REQUIRES failed at save_restore_tensor.cc:182 : Data loss: Unable to open table file /home/jovyan/work/src/output/models: Failed precondition: /home/jovyan/work/src/output/models; Is a directory: perhaps your file is in a different file format and you need to use a different restore operator?
63 2022-06-27T06:36:16.858577056Z Traceback (most recent call last):
64 2022-06-27T06:36:16.858614223Z File "/home/jovyan/.virtualenvs/basenv/lib/python3.7/site-packages/tensorflow/python/client/session.py", line 1365, in _do_call
65 2022-06-27T06:36:16.860723274Z return fn(*args)
66 2022-06-27T06:36:16.860770309Z File "/home/jovyan/.virtualenvs/basenv/lib/python3.7/site-packages/tensorflow/python/client/session.py", line 1350, in _run_fn
67 2022-06-27T06:36:16.860778361Z target_list, run_metadata)
68 2022-06-27T06:36:16.860783251Z File "/home/jovyan/.virtualenvs/basenv/lib/python3.7/site-packages/tensorflow/python/client/session.py", line 1443, in _call_tf_sessionrun
69 2022-06-27T06:36:16.860788557Z run_metadata)
70 2022-06-27T06:36:16.860793314Z tensorflow.python.framework.errors_impl.DataLossError: 2 root error(s) found.
71 2022-06-27T06:36:16.860798308Z (0) Data loss: Unable to open table file /home/jovyan/work/src/output/models: Failed precondition: /home/jovyan/work/src/output/models; Is a directory: perhaps your file is in a different file format and you need to use a different restore operator?
72 2022-06-27T06:36:16.860804505Z [[{{node save/RestoreV2}}]]
73 2022-06-27T06:36:16.860809493Z (1) Data loss: Unable to open table file /home/jovyan/work/src/output/models: Failed precondition: /home/jovyan/work/src/output/models; Is a directory: perhaps your file is in a different file format and you need to use a different restore operator?
74 2022-06-27T06:36:16.860815269Z [[{{node save/RestoreV2}}]]
75 2022-06-27T06:36:16.860819647Z [[save/RestoreV2/_17]]
76 2022-06-27T06:36:16.860824283Z 0 successful operations.
77 2022-06-27T06:36:16.860829198Z 0 derived errors ignored.
78 2022-06-27T06:36:16.860833772Z
79 2022-06-27T06:36:16.860838066Z During handling of the above exception, another exception occurred:
80 2022-06-27T06:36:16.86084295Z
81 2022-06-27T06:36:16.860847246Z Traceback (most recent call last):
82 2022-06-27T06:36:16.860865911Z File "src/test.py", line 33, in <module>
83 2022-06-27T06:36:16.867479466Z saver.restore(sess, model_PATH)
84 2022-06-27T06:36:16.867497042Z File "/home/jovyan/.virtualenvs/basenv/lib/python3.7/site-packages/tensorflow/python/training/saver.py", line 1299, in restore
85 2022-06-27T06:36:16.867501318Z {self.saver_def.filename_tensor_name: save_path})
86 2022-06-27T06:36:16.867504328Z File "/home/jovyan/.virtualenvs/basenv/lib/python3.7/site-packages/tensorflow/python/client/session.py", line 958, in run
87 2022-06-27T06:36:16.867507425Z run_metadata_ptr)
88 2022-06-27T06:36:16.867510248Z File "/home/jovyan/.virtualenvs/basenv/lib/python3.7/site-packages/tensorflow/python/client/session.py", line 1181, in _run
89 2022-06-27T06:36:16.867513326Z feed_dict_tensor, options, run_metadata)
90 2022-06-27T06:36:16.867516126Z File "/home/jovyan/.virtualenvs/basenv/lib/python3.7/site-packages/tensorflow/python/client/session.py", line 1359, in _do_run
91 2022-06-27T06:36:16.867519148Z run_metadata)
92 2022-06-27T06:36:16.867521656Z File "/home/jovyan/.virtualenvs/basenv/lib/python3.7/site-packages/tensorflow/python/client/session.py", line 1384, in _do_call
93 2022-06-27T06:36:16.867524457Z raise type(e)(node_def, op, message)
94 2022-06-27T06:36:16.867527053Z tensorflow.python.framework.errors_impl.DataLossError: 2 root error(s) found.
95 2022-06-27T06:36:16.867529858Z (0) Data loss: Unable to open table file /home/jovyan/work/src/output/models: Failed precondition: /home/jovyan/work/src/output/models; Is a directory: perhaps your file is in a different file format and you need to use a different restore operator?
96 2022-06-27T06:36:16.867532909Z [[node save/RestoreV2 (defined at src/test.py:30) ]]
97 2022-06-27T06:36:16.867535905Z (1) Data loss: Unable to open table file /home/jovyan/work/src/output/models: Failed precondition: /home/jovyan/work/src/output/models; Is a directory: perhaps your file is in a different file format and you need to use a different restore operator?
98 2022-06-27T06:36:16.86753885Z [[node save/RestoreV2 (defined at src/test.py:30) ]]
99 2022-06-27T06:36:16.867541735Z [[save/RestoreV2/_17]]
100 2022-06-27T06:36:16.86754431Z 0 successful operations.
101 2022-06-27T06:36:16.867547008Z 0 derived errors ignored.
102 2022-06-27T06:36:16.86754958Z
103 2022-06-27T06:36:16.8675522Z Original stack trace for 'save/RestoreV2':
104 2022-06-27T06:36:16.867554898Z File "src/test.py", line 30, in <module>
105 2022-06-27T06:36:16.86755794Z saver = tf.train.Saver()
106 2022-06-27T06:36:16.867560564Z File "/home/jovyan/.virtualenvs/basenv/lib/python3.7/site-packages/tensorflow/python/training/saver.py", line 836, in __init__
107 2022-06-27T06:36:16.867563425Z self.build()
108 2022-06-27T06:36:16.867565938Z File "/home/jovyan/.virtualenvs/basenv/lib/python3.7/site-packages/tensorflow/python/training/saver.py", line 848, in build
109 2022-06-27T06:36:16.867580705Z self._build(self._filename, build_save=True, build_restore=True)
110 2022-06-27T06:36:16.867583844Z File "/home/jovyan/.virtualenvs/basenv/lib/python3.7/site-packages/tensorflow/python/training/saver.py", line 886, in _build
111 2022-06-27T06:36:16.867586717Z build_restore=build_restore)
112 2022-06-27T06:36:16.86758929Z File "/home/jovyan/.virtualenvs/basenv/lib/python3.7/site-packages/tensorflow/python/training/saver.py", line 516, in _build_internal
113 2022-06-27T06:36:16.867592137Z restore_sequentially, reshape)
114 2022-06-27T06:36:16.867594694Z File "/home/jovyan/.virtualenvs/basenv/lib/python3.7/site-packages/tensorflow/python/training/saver.py", line 336, in _AddRestoreOps
115 2022-06-27T06:36:16.867597486Z restore_sequentially)
116 2022-06-27T06:36:16.867600011Z File "/home/jovyan/.virtualenvs/basenv/lib/python3.7/site-packages/tensorflow/python/training/saver.py", line 583, in bulk_restore
117 2022-06-27T06:36:16.867602824Z return io_ops.restore_v2(filename_tensor, names, slices, dtypes)
118 2022-06-27T06:36:16.867605422Z File "/home/jovyan/.virtualenvs/basenv/lib/python3.7/site-packages/tensorflow/python/ops/gen_io_ops.py", line 1524, in restore_v2
119 2022-06-27T06:36:16.867608998Z name=name)
120 2022-06-27T06:36:16.867611624Z File "/home/jovyan/.virtualenvs/basenv/lib/python3.7/site-packages/tensorflow/python/framework/op_def_library.py", line 744, in _apply_op_helper
121 2022-06-27T06:36:16.867614426Z attrs=attr_protos, op_def=op_def)
122 2022-06-27T06:36:16.867616967Z File "/home/jovyan/.virtualenvs/basenv/lib/python3.7/site-packages/tensorflow/python/framework/ops.py", line 3485, in _create_op_internal
123 2022-06-27T06:36:16.8676198Z op_def=op_def)
124 2022-06-27T06:36:16.867622461Z File "/home/jovyan/.virtualenvs/basenv/lib/python3.7/site-packages/tensorflow/python/framework/ops.py", line 1949, in __init__
125 2022-06-27T06:36:16.867625257Z self._traceback = tf_stack.extract_stack()
126 2022-06-27T06:36:16.86762792Z
127 2022-06-27T06:36:17.563235506Z SYSTEM: Finishing...
+67
-0
job_logs/job-gpu-62b95050f752e3e25d1d3e32.log less more
0 2022-06-27T06:38:15.394534065Z SYSTEM: Preparing env...
1 2022-06-27T06:38:15.936080766Z SYSTEM: Running...
2 2022-06-27T06:38:16.723513366Z 2022-06-27 14:38:16.723151: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
3 2022-06-27T06:38:18.316735193Z WARNING:tensorflow:From /home/jovyan/.virtualenvs/basenv/lib/python3.7/site-packages/tensorflow/python/compat/v2_compat.py:96: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version.
4 2022-06-27T06:38:18.316789541Z Instructions for updating:
5 2022-06-27T06:38:18.316798297Z non-resource variables are not supported in the long term
6 2022-06-27T06:38:18.323267476Z Imported model (for Places365, 128x128 images)
7 2022-06-27T06:38:21.064529748Z WARNING:tensorflow:From /home/jovyan/work/src/model.py:20: conv2d (from tensorflow.python.keras.legacy_tf_layers.convolutional) is deprecated and will be removed in a future version.
8 2022-06-27T06:38:21.064570688Z Instructions for updating:
9 2022-06-27T06:38:21.064600199Z Use `tf.keras.layers.Conv2D` instead.
10 2022-06-27T06:38:21.076688641Z WARNING:tensorflow:From /home/jovyan/.virtualenvs/basenv/lib/python3.7/site-packages/tensorflow/python/keras/legacy_tf_layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer_v1) is deprecated and will be removed in a future version.
11 2022-06-27T06:38:21.076716602Z Instructions for updating:
12 2022-06-27T06:38:21.076724171Z Please use `layer.__call__` method instead.
13 2022-06-27T06:38:21.221472244Z WARNING:tensorflow:From /home/jovyan/work/src/model.py:79: conv2d_transpose (from tensorflow.python.keras.legacy_tf_layers.convolutional) is deprecated and will be removed in a future version.
14 2022-06-27T06:38:21.221498205Z Instructions for updating:
15 2022-06-27T06:38:21.221506439Z Use `tf.keras.layers.Conv2DTranspose` instead.
16 2022-06-27T06:38:21.312483203Z 2022-06-27 14:38:21.311776: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcuda.so.1
17 2022-06-27T06:38:21.356502814Z 2022-06-27 14:38:21.355592: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
18 2022-06-27T06:38:21.361476016Z 2022-06-27 14:38:21.356582: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1716] Found device 0 with properties:
19 2022-06-27T06:38:21.36149573Z pciBusID: 0000:00:08.0 name: Tesla P100-PCIE-16GB computeCapability: 6.0
20 2022-06-27T06:38:21.36151434Z coreClock: 1.3285GHz coreCount: 56 deviceMemorySize: 7.75GiB deviceMemoryBandwidth: 681.88GiB/s
21 2022-06-27T06:38:21.361522652Z 2022-06-27 14:38:21.356626: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
22 2022-06-27T06:38:21.36152877Z 2022-06-27 14:38:21.359432: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcublas.so.10
23 2022-06-27T06:38:21.373639957Z 2022-06-27 14:38:21.362305: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcufft.so.10
24 2022-06-27T06:38:21.373676623Z 2022-06-27 14:38:21.362747: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcurand.so.10
25 2022-06-27T06:38:21.373685101Z 2022-06-27 14:38:21.365718: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusolver.so.10
26 2022-06-27T06:38:21.373690705Z 2022-06-27 14:38:21.367188: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusparse.so.10
27 2022-06-27T06:38:21.387364092Z 2022-06-27 14:38:21.374139: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudnn.so.7
28 2022-06-27T06:38:21.387383972Z 2022-06-27 14:38:21.374593: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
29 2022-06-27T06:38:21.387390216Z 2022-06-27 14:38:21.376159: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
30 2022-06-27T06:38:21.387406699Z 2022-06-27 14:38:21.377495: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1858] Adding visible gpu devices: 0
31 2022-06-27T06:38:21.387412376Z 2022-06-27 14:38:21.378494: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN)to use the following CPU instructions in performance-critical operations: AVX2 FMA
32 2022-06-27T06:38:21.387418019Z To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
33 2022-06-27T06:38:21.431497702Z 2022-06-27 14:38:21.431103: I tensorflow/core/platform/profile_utils/cpu_utils.cc:104] CPU Frequency: 2499990000 Hz
34 2022-06-27T06:38:21.432398121Z 2022-06-27 14:38:21.432015: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x5e57cd0 initialized for platform Host (this does not guarantee that XLA will be used). Devices:
35 2022-06-27T06:38:21.432415269Z 2022-06-27 14:38:21.432102: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version
36 2022-06-27T06:38:21.657683854Z 2022-06-27 14:38:21.654246: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
37 2022-06-27T06:38:21.657726491Z 2022-06-27 14:38:21.655356: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x4efe4c0 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:
38 2022-06-27T06:38:21.657733833Z 2022-06-27 14:38:21.655398: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla P100-PCIE-16GB, Compute Capability 6.0
39 2022-06-27T06:38:21.657739026Z 2022-06-27 14:38:21.655783: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
40 2022-06-27T06:38:21.657757214Z 2022-06-27 14:38:21.656944: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1716] Found device 0 with properties:
41 2022-06-27T06:38:21.657780321Z pciBusID: 0000:00:08.0 name: Tesla P100-PCIE-16GB computeCapability: 6.0
42 2022-06-27T06:38:21.6577862Z coreClock: 1.3285GHz coreCount: 56 deviceMemorySize: 7.75GiB deviceMemoryBandwidth: 681.88GiB/s
43 2022-06-27T06:38:21.657790652Z 2022-06-27 14:38:21.656994: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
44 2022-06-27T06:38:21.65779509Z 2022-06-27 14:38:21.657030: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcublas.so.10
45 2022-06-27T06:38:21.65779984Z 2022-06-27 14:38:21.657057: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcufft.so.10
46 2022-06-27T06:38:21.657804356Z 2022-06-27 14:38:21.657082: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcurand.so.10
47 2022-06-27T06:38:21.657808837Z 2022-06-27 14:38:21.657104: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusolver.so.10
48 2022-06-27T06:38:21.657813355Z 2022-06-27 14:38:21.657126: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusparse.so.10
49 2022-06-27T06:38:21.65781946Z 2022-06-27 14:38:21.657149: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudnn.so.7
50 2022-06-27T06:38:21.657823716Z 2022-06-27 14:38:21.657381: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
51 2022-06-27T06:38:21.660267085Z 2022-06-27 14:38:21.660041: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
52 2022-06-27T06:38:21.668181623Z 2022-06-27 14:38:21.661956: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1858] Adding visible gpu devices: 0
53 2022-06-27T06:38:21.668216811Z 2022-06-27 14:38:21.662074: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
54 2022-06-27T06:38:22.353724159Z 2022-06-27 14:38:22.351212: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1257] Device interconnect StreamExecutor with strength 1 edge matrix:
55 2022-06-27T06:38:22.35376374Z 2022-06-27 14:38:22.351279: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1263] 0
56 2022-06-27T06:38:22.353768071Z 2022-06-27 14:38:22.351336: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1276] 0: N
57 2022-06-27T06:38:22.364752412Z 2022-06-27 14:38:22.361278: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
58 2022-06-27T06:38:22.364805738Z 2022-06-27 14:38:22.362655: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
59 2022-06-27T06:38:22.364814235Z 2022-06-27 14:38:22.363649: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1402] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 7116 MB memory) -> physical GPU (device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:08.0, compute capability: 6.0)
60 2022-06-27T06:38:22.375506339Z Traceback (most recent call last):
61 2022-06-27T06:38:22.375556853Z File "src/test.py", line 33, in <module>
62 2022-06-27T06:38:22.386370827Z saver.restore(sess, model_PATH)
63 2022-06-27T06:38:22.386415106Z File "/home/jovyan/.virtualenvs/basenv/lib/python3.7/site-packages/tensorflow/python/training/saver.py", line 1291, in restore
64 2022-06-27T06:38:22.386423007Z checkpoint_prefix)
65 2022-06-27T06:38:22.386428006Z ValueError: The passed save_path is not a valid checkpoint: /home/jovyan/work/src/output/models/
66 2022-06-27T06:38:23.059942491Z SYSTEM: Finishing...
+71
-0
job_logs/job-gpu-62b95091a393bd89f5bbaa0e.log less more
0 2022-06-27T06:39:32.227411156Z SYSTEM: Preparing env...
1 2022-06-27T06:39:32.755526021Z SYSTEM: Running...
2 2022-06-27T06:39:33.491524252Z 2022-06-27 14:39:33.491184: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
3 2022-06-27T06:39:35.12676689Z WARNING:tensorflow:From /home/jovyan/.virtualenvs/basenv/lib/python3.7/site-packages/tensorflow/python/compat/v2_compat.py:96: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version.
4 2022-06-27T06:39:35.126797199Z Instructions for updating:
5 2022-06-27T06:39:35.12680371Z non-resource variables are not supported in the long term
6 2022-06-27T06:39:35.131545901Z Imported model (for Places365, 128x128 images)
7 2022-06-27T06:39:37.832567591Z WARNING:tensorflow:From /home/jovyan/work/src/model.py:20: conv2d (from tensorflow.python.keras.legacy_tf_layers.convolutional) is deprecated and will be removed in a future version.
8 2022-06-27T06:39:37.832620614Z Instructions for updating:
9 2022-06-27T06:39:37.832628947Z Use `tf.keras.layers.Conv2D` instead.
10 2022-06-27T06:39:37.838525717Z WARNING:tensorflow:From /home/jovyan/.virtualenvs/basenv/lib/python3.7/site-packages/tensorflow/python/keras/legacy_tf_layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer_v1) is deprecated and will be removed in a future version.
11 2022-06-27T06:39:37.838587273Z Instructions for updating:
12 2022-06-27T06:39:37.838594925Z Please use `layer.__call__` method instead.
13 2022-06-27T06:39:37.969525841Z WARNING:tensorflow:From /home/jovyan/work/src/model.py:79: conv2d_transpose (from tensorflow.python.keras.legacy_tf_layers.convolutional) is deprecated and will be removed in a future version.
14 2022-06-27T06:39:37.969573824Z Instructions for updating:
15 2022-06-27T06:39:37.969580998Z Use `tf.keras.layers.Conv2DTranspose` instead.
16 2022-06-27T06:39:38.049508225Z 2022-06-27 14:39:38.048495: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcuda.so.1
17 2022-06-27T06:39:38.143562415Z 2022-06-27 14:39:38.133245: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
18 2022-06-27T06:39:38.143609396Z 2022-06-27 14:39:38.134162: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1716] Found device 0 with properties:
19 2022-06-27T06:39:38.143614886Z pciBusID: 0000:00:08.0 name: Tesla P100-PCIE-16GB computeCapability: 6.0
20 2022-06-27T06:39:38.143619664Z coreClock: 1.3285GHz coreCount: 56 deviceMemorySize: 7.75GiB deviceMemoryBandwidth: 681.88GiB/s
21 2022-06-27T06:39:38.143624592Z 2022-06-27 14:39:38.134214: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
22 2022-06-27T06:39:38.143629424Z 2022-06-27 14:39:38.136831: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcublas.so.10
23 2022-06-27T06:39:38.14363438Z 2022-06-27 14:39:38.139753: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcufft.so.10
24 2022-06-27T06:39:38.143656537Z 2022-06-27 14:39:38.140190: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcurand.so.10
25 2022-06-27T06:39:38.15524677Z 2022-06-27 14:39:38.143579: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusolver.so.10
26 2022-06-27T06:39:38.15526647Z 2022-06-27 14:39:38.145303: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusparse.so.10
27 2022-06-27T06:39:38.155272765Z 2022-06-27 14:39:38.151591: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudnn.so.7
28 2022-06-27T06:39:38.155278041Z 2022-06-27 14:39:38.151929: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
29 2022-06-27T06:39:38.155313673Z 2022-06-27 14:39:38.153038: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
30 2022-06-27T06:39:38.155320285Z 2022-06-27 14:39:38.154006: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1858] Adding visible gpu devices: 0
31 2022-06-27T06:39:38.155324938Z 2022-06-27 14:39:38.154836: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN)to use the following CPU instructions in performance-critical operations: AVX2 FMA
32 2022-06-27T06:39:38.15533058Z To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
33 2022-06-27T06:39:38.169655627Z 2022-06-27 14:39:38.169268: I tensorflow/core/platform/profile_utils/cpu_utils.cc:104] CPU Frequency: 2499990000 Hz
34 2022-06-27T06:39:38.170861573Z 2022-06-27 14:39:38.170220: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x5c680c0 initialized for platform Host (this does not guarantee that XLA will be used). Devices:
35 2022-06-27T06:39:38.170882428Z 2022-06-27 14:39:38.170264: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version
36 2022-06-27T06:39:38.351589663Z 2022-06-27 14:39:38.351012: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
37 2022-06-27T06:39:38.374522059Z 2022-06-27 14:39:38.352464: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x2b14430 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:
38 2022-06-27T06:39:38.37457054Z 2022-06-27 14:39:38.352532: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla P100-PCIE-16GB, Compute Capability 6.0
39 2022-06-27T06:39:38.37457901Z 2022-06-27 14:39:38.353104: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
40 2022-06-27T06:39:38.374585377Z 2022-06-27 14:39:38.353982: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1716] Found device 0 with properties:
41 2022-06-27T06:39:38.374607731Z pciBusID: 0000:00:08.0 name: Tesla P100-PCIE-16GB computeCapability: 6.0
42 2022-06-27T06:39:38.374614597Z coreClock: 1.3285GHz coreCount: 56 deviceMemorySize: 7.75GiB deviceMemoryBandwidth: 681.88GiB/s
43 2022-06-27T06:39:38.374619062Z 2022-06-27 14:39:38.354019: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
44 2022-06-27T06:39:38.37462404Z 2022-06-27 14:39:38.354052: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcublas.so.10
45 2022-06-27T06:39:38.374628981Z 2022-06-27 14:39:38.354080: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcufft.so.10
46 2022-06-27T06:39:38.374634219Z 2022-06-27 14:39:38.354105: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcurand.so.10
47 2022-06-27T06:39:38.374639131Z 2022-06-27 14:39:38.354129: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusolver.so.10
48 2022-06-27T06:39:38.374644441Z 2022-06-27 14:39:38.354152: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusparse.so.10
49 2022-06-27T06:39:38.374649481Z 2022-06-27 14:39:38.354189: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudnn.so.7
50 2022-06-27T06:39:38.374657131Z 2022-06-27 14:39:38.354423: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
51 2022-06-27T06:39:38.374663194Z 2022-06-27 14:39:38.355564: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
52 2022-06-27T06:39:38.374668509Z 2022-06-27 14:39:38.356746: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1858] Adding visible gpu devices: 0
53 2022-06-27T06:39:38.374673859Z 2022-06-27 14:39:38.356826: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
54 2022-06-27T06:39:39.03151241Z 2022-06-27 14:39:39.030685: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1257] Device interconnect StreamExecutor with strength 1 edge matrix:
55 2022-06-27T06:39:39.031555785Z 2022-06-27 14:39:39.030753: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1263] 0
56 2022-06-27T06:39:39.031562668Z 2022-06-27 14:39:39.030766: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1276] 0: N
57 2022-06-27T06:39:39.041913611Z 2022-06-27 14:39:39.039050: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
58 2022-06-27T06:39:39.041959612Z 2022-06-27 14:39:39.040117: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
59 2022-06-27T06:39:39.041968069Z 2022-06-27 14:39:39.040997: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1402] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 7116 MB memory) -> physical GPU (device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:08.0, compute capability: 6.0)
60 2022-06-27T06:39:39.350472788Z 2022-06-27 14:39:39.345683: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudnn.so.7
61 2022-06-27T06:39:42.628125954Z 2022-06-27 14:39:42.623595: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcublas.so.10
62 2022-06-27T06:39:44.061667112Z Traceback (most recent call last):
63 2022-06-27T06:39:44.061719351Z File "src/test.py", line 35, in <module>
64 2022-06-27T06:39:44.062379741Z util.save_image(output[0], out_PATH)
65 2022-06-27T06:39:44.062396766Z File "/home/jovyan/work/src/util.py", line 89, in save_image
66 2022-06-27T06:39:44.062672574Z img.save(name, format='PNG')
67 2022-06-27T06:39:44.062683885Z File "/home/jovyan/.virtualenvs/basenv/lib/python3.7/site-packages/PIL/Image.py", line 2161, in save
68 2022-06-27T06:39:44.071457371Z fp = builtins.open(filename, "w+b")
69 2022-06-27T06:39:44.071493873Z IsADirectoryError: [Errno 21] Is a directory: '/home/jovyan/work/results/'
70 2022-06-27T06:39:45.129121047Z SYSTEM: Finishing...
+63
-0
job_logs/job-gpu-62b950d6f9c7fbd55d4e9e0f.log less more
0 2022-06-27T06:40:28.664029384Z SYSTEM: Preparing env...
1 2022-06-27T06:40:29.161948578Z SYSTEM: Running...
2 2022-06-27T06:40:29.90655155Z 2022-06-27 14:40:29.905643: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
3 2022-06-27T06:40:31.552840979Z WARNING:tensorflow:From /home/jovyan/.virtualenvs/basenv/lib/python3.7/site-packages/tensorflow/python/compat/v2_compat.py:96: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version.
4 2022-06-27T06:40:31.552884694Z Instructions for updating:
5 2022-06-27T06:40:31.552891704Z non-resource variables are not supported in the long term
6 2022-06-27T06:40:31.557248833Z Imported model (for Places365, 128x128 images)
7 2022-06-27T06:40:34.255714665Z WARNING:tensorflow:From /home/jovyan/work/src/model.py:20: conv2d (from tensorflow.python.keras.legacy_tf_layers.convolutional) is deprecated and will be removed in a future version.
8 2022-06-27T06:40:34.255762327Z Instructions for updating:
9 2022-06-27T06:40:34.255770327Z Use `tf.keras.layers.Conv2D` instead.
10 2022-06-27T06:40:34.255776823Z WARNING:tensorflow:From /home/jovyan/.virtualenvs/basenv/lib/python3.7/site-packages/tensorflow/python/keras/legacy_tf_layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer_v1) is deprecated and will be removed in a future version.
11 2022-06-27T06:40:34.255788806Z Instructions for updating:
12 2022-06-27T06:40:34.255793807Z Please use `layer.__call__` method instead.
13 2022-06-27T06:40:34.385566834Z WARNING:tensorflow:From /home/jovyan/work/src/model.py:79: conv2d_transpose (from tensorflow.python.keras.legacy_tf_layers.convolutional) is deprecated and will be removed in a future version.
14 2022-06-27T06:40:34.385615911Z Instructions for updating:
15 2022-06-27T06:40:34.385623287Z Use `tf.keras.layers.Conv2DTranspose` instead.
16 2022-06-27T06:40:34.46652205Z 2022-06-27 14:40:34.463569: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcuda.so.1
17 2022-06-27T06:40:34.587041885Z 2022-06-27 14:40:34.558264: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
18 2022-06-27T06:40:34.587091586Z 2022-06-27 14:40:34.559496: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1716] Found device 0 with properties:
19 2022-06-27T06:40:34.587099226Z pciBusID: 0000:00:08.0 name: Tesla P100-PCIE-16GB computeCapability: 6.0
20 2022-06-27T06:40:34.587104431Z coreClock: 1.3285GHz coreCount: 56 deviceMemorySize: 7.75GiB deviceMemoryBandwidth: 681.88GiB/s
21 2022-06-27T06:40:34.587110049Z 2022-06-27 14:40:34.559547: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
22 2022-06-27T06:40:34.587115546Z 2022-06-27 14:40:34.563201: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcublas.so.10
23 2022-06-27T06:40:34.587121023Z 2022-06-27 14:40:34.566815: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcufft.so.10
24 2022-06-27T06:40:34.587142859Z 2022-06-27 14:40:34.567434: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcurand.so.10
25 2022-06-27T06:40:34.587149143Z 2022-06-27 14:40:34.571508: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusolver.so.10
26 2022-06-27T06:40:34.587154254Z 2022-06-27 14:40:34.573557: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusparse.so.10
27 2022-06-27T06:40:34.587159273Z 2022-06-27 14:40:34.581702: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudnn.so.7
28 2022-06-27T06:40:34.58717697Z 2022-06-27 14:40:34.582161: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
29 2022-06-27T06:40:34.587184192Z 2022-06-27 14:40:34.583629: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
30 2022-06-27T06:40:34.587191825Z 2022-06-27 14:40:34.584731: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1858] Adding visible gpu devices: 0
31 2022-06-27T06:40:34.587196874Z 2022-06-27 14:40:34.585721: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN)to use the following CPU instructions in performance-critical operations: AVX2 FMA
32 2022-06-27T06:40:34.587202666Z To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
33 2022-06-27T06:40:34.605273814Z 2022-06-27 14:40:34.604881: I tensorflow/core/platform/profile_utils/cpu_utils.cc:104] CPU Frequency: 2499990000 Hz
34 2022-06-27T06:40:34.612440387Z 2022-06-27 14:40:34.611455: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x43ef4a0 initialized for platform Host (this does not guarantee that XLA will be used). Devices:
35 2022-06-27T06:40:34.612469486Z 2022-06-27 14:40:34.611536: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version
36 2022-06-27T06:40:34.818065383Z 2022-06-27 14:40:34.816854: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
37 2022-06-27T06:40:34.818226758Z 2022-06-27 14:40:34.817945: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x1133430 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:
38 2022-06-27T06:40:34.818242687Z 2022-06-27 14:40:34.818059: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla P100-PCIE-16GB, Compute Capability 6.0
39 2022-06-27T06:40:34.819098669Z 2022-06-27 14:40:34.818920: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
40 2022-06-27T06:40:34.826000267Z 2022-06-27 14:40:34.820425: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1716] Found device 0 with properties:
41 2022-06-27T06:40:34.826040277Z pciBusID: 0000:00:08.0 name: Tesla P100-PCIE-16GB computeCapability: 6.0
42 2022-06-27T06:40:34.826047919Z coreClock: 1.3285GHz coreCount: 56 deviceMemorySize: 7.75GiB deviceMemoryBandwidth: 681.88GiB/s
43 2022-06-27T06:40:34.82605344Z 2022-06-27 14:40:34.820483: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
44 2022-06-27T06:40:34.82605896Z 2022-06-27 14:40:34.820538: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcublas.so.10
45 2022-06-27T06:40:34.826064758Z 2022-06-27 14:40:34.820589: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcufft.so.10
46 2022-06-27T06:40:34.826069913Z 2022-06-27 14:40:34.820637: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcurand.so.10
47 2022-06-27T06:40:34.826075151Z 2022-06-27 14:40:34.820683: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusolver.so.10
48 2022-06-27T06:40:34.826080279Z 2022-06-27 14:40:34.820757: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusparse.so.10
49 2022-06-27T06:40:34.826085367Z 2022-06-27 14:40:34.820807: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudnn.so.7
50 2022-06-27T06:40:34.826090711Z 2022-06-27 14:40:34.821073: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
51 2022-06-27T06:40:34.8260964Z 2022-06-27 14:40:34.822732: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
52 2022-06-27T06:40:34.826101603Z 2022-06-27 14:40:34.824064: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1858] Adding visible gpu devices: 0
53 2022-06-27T06:40:34.826106851Z 2022-06-27 14:40:34.824140: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
54 2022-06-27T06:40:35.48051732Z 2022-06-27 14:40:35.478735: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1257] Device interconnect StreamExecutor with strength 1 edge matrix:
55 2022-06-27T06:40:35.480572834Z 2022-06-27 14:40:35.478800: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1263] 0
56 2022-06-27T06:40:35.480580695Z 2022-06-27 14:40:35.478813: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1276] 0: N
57 2022-06-27T06:40:35.491016804Z 2022-06-27 14:40:35.488592: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
58 2022-06-27T06:40:35.492925518Z 2022-06-27 14:40:35.491127: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
59 2022-06-27T06:40:35.492963862Z 2022-06-27 14:40:35.492080: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1402] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 7116 MB memory) -> physical GPU (device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:08.0, compute capability: 6.0)
60 2022-06-27T06:40:35.830526984Z 2022-06-27 14:40:35.825663: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudnn.so.7
61 2022-06-27T06:40:37.078582913Z 2022-06-27 14:40:37.071379: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcublas.so.10
62 2022-06-27T06:40:39.337113298Z SYSTEM: Finishing...
+63
-0
job_logs/job-gpu-62b96ec3f901e7972521f6bc.log less more
0 2022-06-27T08:48:09.24239905Z SYSTEM: Preparing env...
1 2022-06-27T08:48:09.786941439Z SYSTEM: Running...
2 2022-06-27T08:48:10.51858763Z 2022-06-27 16:48:10.517635: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
3 2022-06-27T08:48:12.178056383Z WARNING:tensorflow:From /home/jovyan/.virtualenvs/basenv/lib/python3.7/site-packages/tensorflow/python/compat/v2_compat.py:96: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version.
4 2022-06-27T08:48:12.178109572Z Instructions for updating:
5 2022-06-27T08:48:12.178117458Z non-resource variables are not supported in the long term
6 2022-06-27T08:48:12.183152171Z Imported model (for Places365, 128x128 images)
7 2022-06-27T08:48:14.874564815Z WARNING:tensorflow:From /home/jovyan/work/src/model.py:20: conv2d (from tensorflow.python.keras.legacy_tf_layers.convolutional) is deprecated and will be removed in a future version.
8 2022-06-27T08:48:14.87460536Z Instructions for updating:
9 2022-06-27T08:48:14.874612991Z Use `tf.keras.layers.Conv2D` instead.
10 2022-06-27T08:48:14.879516836Z WARNING:tensorflow:From /home/jovyan/.virtualenvs/basenv/lib/python3.7/site-packages/tensorflow/python/keras/legacy_tf_layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer_v1) is deprecated and will be removed in a future version.
11 2022-06-27T08:48:14.879552898Z Instructions for updating:
12 2022-06-27T08:48:14.879556966Z Please use `layer.__call__` method instead.
13 2022-06-27T08:48:14.998783159Z WARNING:tensorflow:From /home/jovyan/work/src/model.py:79: conv2d_transpose (from tensorflow.python.keras.legacy_tf_layers.convolutional) is deprecated and will be removed in a future version.
14 2022-06-27T08:48:14.99882407Z Instructions for updating:
15 2022-06-27T08:48:14.998827896Z Use `tf.keras.layers.Conv2DTranspose` instead.
16 2022-06-27T08:48:15.082132374Z 2022-06-27 16:48:15.075591: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcuda.so.1
17 2022-06-27T08:48:15.167539823Z 2022-06-27 16:48:15.154770: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
18 2022-06-27T08:48:15.167590578Z 2022-06-27 16:48:15.156152: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1716] Found device 0 with properties:
19 2022-06-27T08:48:15.16759558Z pciBusID: 0000:00:08.0 name: Tesla P100-PCIE-16GB computeCapability: 6.0
20 2022-06-27T08:48:15.167598782Z coreClock: 1.3285GHz coreCount: 56 deviceMemorySize: 7.75GiB deviceMemoryBandwidth: 681.88GiB/s
21 2022-06-27T08:48:15.167602013Z 2022-06-27 16:48:15.156213: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
22 2022-06-27T08:48:15.167605072Z 2022-06-27 16:48:15.160218: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcublas.so.10
23 2022-06-27T08:48:15.167608115Z 2022-06-27 16:48:15.163802: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcufft.so.10
24 2022-06-27T08:48:15.167628968Z 2022-06-27 16:48:15.164231: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcurand.so.10
25 2022-06-27T08:48:15.172464872Z 2022-06-27 16:48:15.167436: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusolver.so.10
26 2022-06-27T08:48:15.17248651Z 2022-06-27 16:48:15.168987: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusparse.so.10
27 2022-06-27T08:48:15.179400732Z 2022-06-27 16:48:15.175372: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudnn.so.7
28 2022-06-27T08:48:15.179421972Z 2022-06-27 16:48:15.175712: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
29 2022-06-27T08:48:15.179428301Z 2022-06-27 16:48:15.176844: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
30 2022-06-27T08:48:15.17943329Z 2022-06-27 16:48:15.177689: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1858] Adding visible gpu devices: 0
31 2022-06-27T08:48:15.179438827Z 2022-06-27 16:48:15.178507: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN)to use the following CPU instructions in performance-critical operations: AVX2 FMA
32 2022-06-27T08:48:15.179443601Z To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
33 2022-06-27T08:48:15.189791017Z 2022-06-27 16:48:15.189490: I tensorflow/core/platform/profile_utils/cpu_utils.cc:104] CPU Frequency: 2499990000 Hz
34 2022-06-27T08:48:15.190826889Z 2022-06-27 16:48:15.190353: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x5d4d3b0 initialized for platform Host (this does not guarantee that XLA will be used). Devices:
35 2022-06-27T08:48:15.190840436Z 2022-06-27 16:48:15.190405: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version
36 2022-06-27T08:48:15.354103229Z 2022-06-27 16:48:15.352890: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
37 2022-06-27T08:48:15.360495813Z 2022-06-27 16:48:15.353962: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x2a91430 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:
38 2022-06-27T08:48:15.36051428Z 2022-06-27 16:48:15.354039: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla P100-PCIE-16GB, Compute Capability 6.0
39 2022-06-27T08:48:15.360518449Z 2022-06-27 16:48:15.354703: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
40 2022-06-27T08:48:15.360521962Z 2022-06-27 16:48:15.355674: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1716] Found device 0 with properties:
41 2022-06-27T08:48:15.360540021Z pciBusID: 0000:00:08.0 name: Tesla P100-PCIE-16GB computeCapability: 6.0
42 2022-06-27T08:48:15.360543183Z coreClock: 1.3285GHz coreCount: 56 deviceMemorySize: 7.75GiB deviceMemoryBandwidth: 681.88GiB/s
43 2022-06-27T08:48:15.360546053Z 2022-06-27 16:48:15.355713: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
44 2022-06-27T08:48:15.360548859Z 2022-06-27 16:48:15.355757: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcublas.so.10
45 2022-06-27T08:48:15.360551628Z 2022-06-27 16:48:15.355789: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcufft.so.10
46 2022-06-27T08:48:15.360554374Z 2022-06-27 16:48:15.355815: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcurand.so.10
47 2022-06-27T08:48:15.360557367Z 2022-06-27 16:48:15.355840: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusolver.so.10
48 2022-06-27T08:48:15.360560106Z 2022-06-27 16:48:15.355863: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusparse.so.10
49 2022-06-27T08:48:15.360562889Z 2022-06-27 16:48:15.355889: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudnn.so.7
50 2022-06-27T08:48:15.360566266Z 2022-06-27 16:48:15.356094: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
51 2022-06-27T08:48:15.360569337Z 2022-06-27 16:48:15.357167: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
52 2022-06-27T08:48:15.360572245Z 2022-06-27 16:48:15.358197: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1858] Adding visible gpu devices: 0
53 2022-06-27T08:48:15.360574967Z 2022-06-27 16:48:15.358256: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
54 2022-06-27T08:48:16.025531234Z 2022-06-27 16:48:16.024309: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1257] Device interconnect StreamExecutor with strength 1 edge matrix:
55 2022-06-27T08:48:16.02556822Z 2022-06-27 16:48:16.024381: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1263] 0
56 2022-06-27T08:48:16.02557269Z 2022-06-27 16:48:16.024394: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1276] 0: N
57 2022-06-27T08:48:16.036018785Z 2022-06-27 16:48:16.033186: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
58 2022-06-27T08:48:16.036045308Z 2022-06-27 16:48:16.034241: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
59 2022-06-27T08:48:16.036052087Z 2022-06-27 16:48:16.035136: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1402] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 7116 MB memory) -> physical GPU (device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:08.0, compute capability: 6.0)
60 2022-06-27T08:48:16.324801304Z 2022-06-27 16:48:16.323386: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudnn.so.7
61 2022-06-27T08:48:17.556743331Z 2022-06-27 16:48:17.551710: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcublas.so.10
62 2022-06-27T08:48:19.783798978Z SYSTEM: Finishing...
+15
-0
job_logs/job-gpu-62b9701b8029151df74612ce.log less more
0 2022-06-27T08:53:53.960313457Z SYSTEM: Preparing env...
1 2022-06-27T08:53:54.530950342Z SYSTEM: Running...
2 2022-06-27T08:53:55.275717821Z 2022-06-27 16:53:55.274909: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
3 2022-06-27T08:53:56.910131992Z WARNING:tensorflow:From /home/jovyan/.virtualenvs/basenv/lib/python3.7/site-packages/tensorflow/python/compat/v2_compat.py:96: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version.
4 2022-06-27T08:53:56.910179395Z Instructions for updating:
5 2022-06-27T08:53:56.91018644Z non-resource variables are not supported in the long term
6 2022-06-27T08:53:56.915074142Z Imported model (for Places365, 128x128 images)
7 2022-06-27T08:53:59.436145133Z Traceback (most recent call last):
8 2022-06-27T08:53:59.436188131Z File "src/test.py", line 26, in <module>
9 2022-06-27T08:53:59.436409259Z img_p = util.preprocess_images_outpainting(img)
10 2022-06-27T08:53:59.43642622Z File "/home/jovyan/work/src/util.py", line 58, in preprocess_images_outpainting
11 2022-06-27T08:53:59.436715361Z imgs_p = np.concatenate((imgs, mask), axis=3)
12 2022-06-27T08:53:59.436729791Z File "<__array_function__ internals>", line 6, in concatenate
13 2022-06-27T08:53:59.443505289Z ValueError: all the input array dimensions for the concatenation axis must match exactly, but along dimension 1, the array at index 0 has size 250 and the array at index 1 has size 128
14 2022-06-27T08:53:59.902211373Z SYSTEM: Finishing...
+63
-0
job_logs/job-gpu-62b971036452cd65a61ca625.log less more
0 2022-06-27T08:57:46.196243927Z SYSTEM: Preparing env...
1 2022-06-27T08:57:46.729899376Z SYSTEM: Running...
2 2022-06-27T08:57:47.457595515Z 2022-06-27 16:57:47.456138: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
3 2022-06-27T08:57:49.146821587Z WARNING:tensorflow:From /home/jovyan/.virtualenvs/basenv/lib/python3.7/site-packages/tensorflow/python/compat/v2_compat.py:96: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version.
4 2022-06-27T08:57:49.146871573Z Instructions for updating:
5 2022-06-27T08:57:49.146880336Z non-resource variables are not supported in the long term
6 2022-06-27T08:57:49.151937708Z Imported model (for Places365, 128x128 images)
7 2022-06-27T08:57:51.925523771Z WARNING:tensorflow:From /home/jovyan/work/src/model.py:20: conv2d (from tensorflow.python.keras.legacy_tf_layers.convolutional) is deprecated and will be removed in a future version.
8 2022-06-27T08:57:51.925574935Z Instructions for updating:
9 2022-06-27T08:57:51.925582982Z Use `tf.keras.layers.Conv2D` instead.
10 2022-06-27T08:57:51.938638561Z WARNING:tensorflow:From /home/jovyan/.virtualenvs/basenv/lib/python3.7/site-packages/tensorflow/python/keras/legacy_tf_layers/convolutional.py:424: Layer.apply (from tensorflow.python.keras.engine.base_layer_v1) is deprecated and will be removed in a future version.
11 2022-06-27T08:57:51.938691742Z Instructions for updating:
12 2022-06-27T08:57:51.938700174Z Please use `layer.__call__` method instead.
13 2022-06-27T08:57:52.053507181Z WARNING:tensorflow:From /home/jovyan/work/src/model.py:79: conv2d_transpose (from tensorflow.python.keras.legacy_tf_layers.convolutional) is deprecated and will be removed in a future version.
14 2022-06-27T08:57:52.053542901Z Instructions for updating:
15 2022-06-27T08:57:52.053549331Z Use `tf.keras.layers.Conv2DTranspose` instead.
16 2022-06-27T08:57:52.137499385Z 2022-06-27 16:57:52.133265: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcuda.so.1
17 2022-06-27T08:57:52.241583163Z 2022-06-27 16:57:52.233707: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
18 2022-06-27T08:57:52.241641201Z 2022-06-27 16:57:52.234628: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1716] Found device 0 with properties:
19 2022-06-27T08:57:52.241649804Z pciBusID: 0000:00:08.0 name: Tesla P100-PCIE-16GB computeCapability: 6.0
20 2022-06-27T08:57:52.241654633Z coreClock: 1.3285GHz coreCount: 56 deviceMemorySize: 7.75GiB deviceMemoryBandwidth: 681.88GiB/s
21 2022-06-27T08:57:52.241659849Z 2022-06-27 16:57:52.234668: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
22 2022-06-27T08:57:52.241664684Z 2022-06-27 16:57:52.237458: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcublas.so.10
23 2022-06-27T08:57:52.241669496Z 2022-06-27 16:57:52.240254: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcufft.so.10
24 2022-06-27T08:57:52.241690618Z 2022-06-27 16:57:52.240676: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcurand.so.10
25 2022-06-27T08:57:52.250526608Z 2022-06-27 16:57:52.243841: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusolver.so.10
26 2022-06-27T08:57:52.250549004Z 2022-06-27 16:57:52.245384: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusparse.so.10
27 2022-06-27T08:57:52.255036662Z 2022-06-27 16:57:52.251546: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudnn.so.7
28 2022-06-27T08:57:52.255059262Z 2022-06-27 16:57:52.251873: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
29 2022-06-27T08:57:52.255067412Z 2022-06-27 16:57:52.252957: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
30 2022-06-27T08:57:52.255073068Z 2022-06-27 16:57:52.253796: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1858] Adding visible gpu devices: 0
31 2022-06-27T08:57:52.25507835Z 2022-06-27 16:57:52.254586: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN)to use the following CPU instructions in performance-critical operations: AVX2 FMA
32 2022-06-27T08:57:52.255083681Z To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.
33 2022-06-27T08:57:52.271420544Z 2022-06-27 16:57:52.270498: I tensorflow/core/platform/profile_utils/cpu_utils.cc:104] CPU Frequency: 2499990000 Hz
34 2022-06-27T08:57:52.271743118Z 2022-06-27 16:57:52.271455: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x4747740 initialized for platform Host (this does not guarantee that XLA will be used). Devices:
35 2022-06-27T08:57:52.271767039Z 2022-06-27 16:57:52.271509: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Host, Default Version
36 2022-06-27T08:57:52.444513161Z 2022-06-27 16:57:52.438561: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
37 2022-06-27T08:57:52.444560328Z 2022-06-27 16:57:52.439606: I tensorflow/compiler/xla/service/service.cc:168] XLA service 0x44724a0 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:
38 2022-06-27T08:57:52.444568583Z 2022-06-27 16:57:52.439644: I tensorflow/compiler/xla/service/service.cc:176] StreamExecutor device (0): Tesla P100-PCIE-16GB, Compute Capability 6.0
39 2022-06-27T08:57:52.444573851Z 2022-06-27 16:57:52.440010: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
40 2022-06-27T08:57:52.444579411Z 2022-06-27 16:57:52.440946: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1716] Found device 0 with properties:
41 2022-06-27T08:57:52.444601594Z pciBusID: 0000:00:08.0 name: Tesla P100-PCIE-16GB computeCapability: 6.0
42 2022-06-27T08:57:52.444607895Z coreClock: 1.3285GHz coreCount: 56 deviceMemorySize: 7.75GiB deviceMemoryBandwidth: 681.88GiB/s
43 2022-06-27T08:57:52.44461265Z 2022-06-27 16:57:52.440984: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
44 2022-06-27T08:57:52.444617514Z 2022-06-27 16:57:52.441015: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcublas.so.10
45 2022-06-27T08:57:52.444622224Z 2022-06-27 16:57:52.441041: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcufft.so.10
46 2022-06-27T08:57:52.444627069Z 2022-06-27 16:57:52.441064: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcurand.so.10
47 2022-06-27T08:57:52.444631482Z 2022-06-27 16:57:52.441085: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusolver.so.10
48 2022-06-27T08:57:52.44463649Z 2022-06-27 16:57:52.441106: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcusparse.so.10
49 2022-06-27T08:57:52.444642981Z 2022-06-27 16:57:52.441130: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudnn.so.7
50 2022-06-27T08:57:52.444648137Z 2022-06-27 16:57:52.441319: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
51 2022-06-27T08:57:52.444653345Z 2022-06-27 16:57:52.442307: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
52 2022-06-27T08:57:52.444658464Z 2022-06-27 16:57:52.443119: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1858] Adding visible gpu devices: 0
53 2022-06-27T08:57:52.444662832Z 2022-06-27 16:57:52.443165: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudart.so.10.1
54 2022-06-27T08:57:53.113557514Z 2022-06-27 16:57:53.112857: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1257] Device interconnect StreamExecutor with strength 1 edge matrix:
55 2022-06-27T08:57:53.113618014Z 2022-06-27 16:57:53.112923: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1263] 0
56 2022-06-27T08:57:53.113626372Z 2022-06-27 16:57:53.112936: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1276] 0: N
57 2022-06-27T08:57:53.125264805Z 2022-06-27 16:57:53.122189: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
58 2022-06-27T08:57:53.125308939Z 2022-06-27 16:57:53.123241: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:982] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero
59 2022-06-27T08:57:53.125317622Z 2022-06-27 16:57:53.124239: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1402] Created TensorFlow device (/job:localhost/replica:0/task:0/device:GPU:0 with 7116 MB memory) -> physical GPU (device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:08.0, compute capability: 6.0)
60 2022-06-27T08:57:53.434540742Z 2022-06-27 16:57:53.425442: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcudnn.so.7
61 2022-06-27T08:57:54.72057696Z 2022-06-27 16:57:54.713817: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library libcublas.so.10
62 2022-06-27T08:57:57.027310115Z SYSTEM: Finishing...
poster/.ipynb_checkpoints/msabini-gili__image-outpainting-poster-checkpoint.pdf less more
Binary diff not shown
+140
-0
project_requirements.txt less more
0 murmurhash==1.0.6
1 jmespath==0.10.0
2 configparser==5.2.0
3 imbalanced-learn==0.6.2
4 Augmentor==0.2.8
5 scikit-image==0.15.0
6 tensorboard-data-server==0.6.1
7 requests-oauthlib==1.3.1
8 argon2-cffi-bindings==21.2.0
9 thinc==7.4.1
10 charset-normalizer==2.0.12
11 nltk==3.5
12 transformers==4.1.1
13 astunparse==1.6.3
14 PyWavelets==1.2.0
15 semantic-version==2.8.5
16 sentencepiece==0.1.91
17 PyAudio==0.2.11
18 greenlet==1.1.2
19 ruamel.yaml.clib==0.2.6
20 pyasn1==0.4.8
21 attrs==19.3.0
22 retrying==1.3.3
23 torchtext==0.6.0
24 func-timeout==4.3.5
25 gym==0.15.7
26 importlib-metadata==4.11.1
27 pyasn1-modules==0.2.8
28 cmake==3.21.1
29 word2vec==0.11.1
30 packaging==21.3
31 preshed==3.0.6
32 regex==2022.1.18
33 tensorflow-privacy==0.5.2
34 rsa==4.8
35 easydict==1.9
36 spacy==2.3.2
37 tensorboardX==2.0
38 defusedxml==0.7.1
39 s3transfer==0.3.3
40 networkx==2.6.3
41 catalogue==1.0.0
42 openpyxl==2.6.4
43 dm-tree==0.1.6
44 imageio==2.8.0
45 metakernel==0.28.2
46 opt-einsum==3.3.0
47 cachetools==3.1.1
48 smart-open==5.2.1
49 multipledispatch==0.6.0
50 argon2-cffi==21.3.0
51 tinycss2==1.1.1
52 graphviz==0.14
53 minio==5.0.10
54 boto3==1.16.25
55 unification==0.2.2
56 oauthlib==3.2.0
57 google-auth==2.6.0
58 gensim==3.8.3
59 tensorboard-plugin-wit==1.8.1
60 pytorch-pretrained-bert==0.6.2
61 plotly==4.8.1
62 joblib==1.1.0
63 pydot==1.4.1
64 sacremoses==0.0.47
65 calysto==1.0.6
66 mpmath==1.2.1
67 toolz==0.11.2
68 en-core-web-sm==https://files.momodel.cn/en_core_web_sm-2.3.0.tar.gz
69 rouge==1.0.0
70 plac==1.1.3
71 importlib-resources==5.4.0
72 et-xmlfile==1.1.0
73 typing-extensions==4.1.1
74 certipy==0.1.3
75 platformdirs==2.5.1
76 debugpy==1.5.1
77 kanren==0.2.3
78 cryptography==36.0.1
79 python-json-logger==2.0.2
80 ruamel.yaml==0.17.21
81 pyglet==1.5.0
82 cymem==2.0.6
83 Shapely==1.7.0
84 click==8.0.4
85 jdcal==1.4.1
86 jupyter-telemetry==0.1.0
87 tensorflow-federated==0.17.0
88 tensorflow-estimator==2.3.0
89 cloudpickle==1.2.2
90 jupyterlab-server==0.2.0
91 nest-asyncio==1.5.4
92 mindspore==https://ms-release.obs.cn-north-4.myhuaweicloud.com/1.0.0/MindSpore/cpu/ubuntu_x86/mindspore-1.0.0-cp37-cp37m-linux_x86_64.whl
93 google-pasta==0.2.0
94 async-generator==1.10
95 yellowbrick==1.1
96 tf-slim==1.1.0
97 xlrd==1.2.0
98 numpyencoder==0.3.0
99 copulas==0.3.3
100 torch==1.4.0+cpu
101 typeguard==2.13.3
102 pyOpenSSL==22.0.0
103 matplotlib-inline==0.1.3
104 tqdm==4.46.1
105 torchvision==0.5.0+cpu
106 wrapt==1.13.3
107 google-auth-oauthlib==0.4.6
108 pycparser==2.21
109 filelock==3.6.0
110 botocore==1.19.25
111 XlsxWriter==1.4.3
112 dlib==19.22.0
113 portpicker==1.3.9
114 pytorch-transformers==1.2.0
115 srsly==1.0.5
116 distlib==0.3.4
117 Cython==0.29.20
118 install==1.3.5
119 cssselect2==0.4.1
120 CairoSVG==2.5.2
121 pygame==2.0.1
122 ipdb==0.13.2
123 zipp==3.7.0
124 tensorflow-model-optimization==0.4.1
125 wasabi==0.9.0
126 svgwrite==1.4.1
127 baytune==0.3.12
128 jieba==0.42.1
129 minepy==1.2.4
130 cffi==1.15.0
131 blis==0.4.1
132 paddlepaddle==2.0.1
133 cairocffi==1.3.0
134 tensorflow-addons==0.11.2
135 sympy==1.6.2
136 pyrsistent==0.18.1
137 imgaug==0.4.0
138 asttokens==2.0.5
139 tokenizers==0.9.4
+2
-0
results/.ipynb_checkpoints/README-checkpoint.md less more
0 Please store your training checkpoints or results here
1 请在此处存储 checkpoints 和结果文件⏎
results/.ipynb_checkpoints/city_output-checkpoint.png less more
Binary diff not shown
results/.ipynb_checkpoints/test_output-checkpoint.png less more
Binary diff not shown
results/city_output.png less more
Binary diff not shown
results/test_output.png less more
Binary diff not shown
+130
-0
src/.ipynb_checkpoints/Untitled-checkpoint.ipynb less more
0 {
1 "cells": [
2 {
3 "cell_type": "code",
4 "execution_count": 3,
5 "id": "db0b0a11",
6 "metadata": {},
7 "outputs": [
8 {
9 "name": "stderr",
10 "output_type": "stream",
11 "text": [
12 "2022-06-27 17:55:39.372674: W tensorflow/stream_executor/platform/default/dso_loader.cc:59] Could not load dynamic library 'libcudart.so.10.1'; dlerror: libcudart.so.10.1: cannot open shared object file: No such file or directory\n",
13 "2022-06-27 17:55:39.372709: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n"
14 ]
15 },
16 {
17 "name": "stdout",
18 "output_type": "stream",
19 "text": [
20 "WARNING:tensorflow:From /usr/local/lib/python3.7/dist-packages/tensorflow/python/compat/v2_compat.py:96: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version.\n",
21 "Instructions for updating:\n",
22 "non-resource variables are not supported in the long term\n",
23 "Imported model (for Places365, 128x128 images)\n"
24 ]
25 }
26 ],
27 "source": [
28 "import tensorflow.compat.v1 as tf\n",
29 "tf.disable_v2_behavior()\n",
30 "tf.reset_default_graph()\n",
31 "import numpy as np\n",
32 "from PIL import Image\n",
33 "import model\n",
34 "import util\n",
35 "import os\n",
36 "import sys"
37 ]
38 },
39 {
40 "cell_type": "code",
41 "execution_count": 4,
42 "id": "377355df",
43 "metadata": {},
44 "outputs": [],
45 "source": [
46 "model_PATH='/home/jovyan/work/src/output/models/model2000.ckpt'"
47 ]
48 },
49 {
50 "cell_type": "code",
51 "execution_count": 8,
52 "id": "fd4ec8ee",
53 "metadata": {},
54 "outputs": [],
55 "source": [
56 "def load_demo_image(in_PATH):\n",
57 " img = np.array(Image.open(in_PATH).convert('RGB'))[np.newaxis] / 255.0\n",
58 " img_p = util.preprocess_images_outpainting(img)\n",
59 " return img_p"
60 ]
61 },
62 {
63 "cell_type": "code",
64 "execution_count": 6,
65 "id": "cc76d61f",
66 "metadata": {},
67 "outputs": [],
68 "source": [
69 "def inference(model_PATH, img_p):\n",
70 " G_Z = tf.placeholder(tf.float32, shape=[None, IMAGE_SZ, IMAGE_SZ, 4], name='G_Z')\n",
71 " G_sample = model.generator(G_Z)\n",
72 " \n",
73 " saver = tf.train.Saver()\n",
74 " with tf.Session() as sess:\n",
75 " saver.restore(sess, model_PATH)\n",
76 " output, = sess.run([G_sample], feed_dict={G_Z: img_p})\n",
77 " img_norm = (output[0] * 255.0).astype(np.uint8)\n",
78 " img = Image.fromarray(img_norm, 'RGB')\n",
79 " #util.save_image(output[0], out_PATH)\n",
80 " return img"
81 ]
82 },
83 {
84 "cell_type": "code",
85 "execution_count": null,
86 "id": "d93edf59",
87 "metadata": {},
88 "outputs": [],
89 "source": [
90 "def handle(conf):\n",
91 " \"\"\"\n",
92 " 该方法是部署之后，其他人调用你的服务时候的处理方法。\n",
93 " 请按规范填写参数结构，这样我们就能替你自动生成配置文件，方便其他人的调用。\n",
94 " 范例：\n",
95 " params['key'] = value # value_type: str # description: some description\n",
96 " value_type 可以选择：img, video, audio, str, int, float, [int], [str], [float]\n",
97 " 参数请放到params字典中，我们会自动解析该变量。\n",
98 " \"\"\"\n",
99 " base64_str = conf['Photo']\n",
100 " image = load_demo_image(base64_str, image_size, device)\n",
101 " res = inference(model, image)\n",
102 " # add your code\n",
103 " return {'Output': res}\n",
104 " "
105 ]
106 }
107 ],
108 "metadata": {
109 "kernelspec": {
110 "display_name": "Python 3 (ipykernel)",
111 "language": "python",
112 "name": "python3"
113 },
114 "language_info": {
115 "codemirror_mode": {
116 "name": "ipython",
117 "version": 3
118 },
119 "file_extension": ".py",
120 "mimetype": "text/x-python",
121 "name": "python",
122 "nbconvert_exporter": "python",
123 "pygments_lexer": "ipython3",
124 "version": "3.7.5"
125 }
126 },
127 "nbformat": 4,
128 "nbformat_minor": 5
129 }
+158
-0
src/.ipynb_checkpoints/model-checkpoint.py less more
0 # proj: image-outpainting
1 # file: model.py
2 # authors: Mark Sabini, Gili Rusak
3 # desc: Model for outpainting on 128x128 images with only
4 # a global discriminator.
5 # -------------------------------------------------------------
6 import tensorflow.compat.v1 as tf
7 tf.disable_v2_behavior()
8
9 print('Imported model (for Places365, 128x128 images)')
10
11 def generator(z):
12 with tf.variable_scope('G', reuse=tf.AUTO_REUSE):
13 conv1 = tf.layers.conv2d(
14 inputs=z,
15 filters=64,
16 kernel_size=[5, 5],
17 strides=(1, 1),
18 padding="same",
19 activation=tf.nn.relu)
20
21 conv2 = tf.layers.conv2d(
22 inputs=conv1,
23 filters=128,
24 kernel_size=[3, 3],
25 strides=(2, 2),
26 padding="same",
27 activation=tf.nn.relu)
28
29 conv3 = tf.layers.conv2d(
30 inputs=conv2,
31 filters=256,
32 kernel_size=[3, 3],
33 strides=(1, 1),
34 padding="same",
35 activation=tf.nn.relu)
36
37 conv4 = tf.layers.conv2d(
38 inputs=conv3,
39 filters=256,
40 kernel_size=[3, 3],
41 strides=(1, 1),
42 dilation_rate=(2, 2),
43 padding="same",
44 activation=tf.nn.relu)
45
46 conv5 = tf.layers.conv2d(
47 inputs=conv4,
48 filters=256,
49 kernel_size=[3, 3],
50 strides=(1, 1),
51 dilation_rate=(4, 4),
52 padding="same",
53 activation=tf.nn.relu)
54
55 conv5_p = tf.layers.conv2d(
56 inputs=conv5,
57 filters=256,
58 kernel_size=[3, 3],
59 strides=(1, 1),
60 dilation_rate=(8, 8),
61 padding="same",
62 activation=tf.nn.relu)
63
64 conv6 = tf.layers.conv2d(
65 inputs=conv5_p,
66 filters=256,
67 kernel_size=[3, 3],
68 strides=(1, 1),
69 padding="same",
70 activation=tf.nn.relu)
71
72 deconv7 = tf.layers.conv2d_transpose(
73 inputs=conv6,
74 filters=128,
75 kernel_size=[4, 4],
76 strides=(2, 2),
77 padding="same",
78 activation=tf.nn.relu)
79
80 conv8 = tf.layers.conv2d(
81 inputs=deconv7,
82 filters=64,
83 kernel_size=[3, 3],
84 strides=(1, 1),
85 padding="same",
86 activation=tf.nn.relu)
87
88 out = tf.layers.conv2d(
89 inputs=conv8,
90 filters=3,
91 kernel_size=[3, 3],
92 strides=(1, 1),
93 padding="same",
94 activation=tf.sigmoid)
95
96 return out
97
98 def global_discriminator(x):
99 with tf.variable_scope('DG', reuse=tf.AUTO_REUSE):
100 conv1 = tf.layers.conv2d(
101 inputs=x,
102 filters=32,
103 kernel_size=[5, 5],
104 strides=(2, 2),
105 padding="same",
106 activation=tf.nn.relu)
107
108 conv2 = tf.layers.conv2d(
109 inputs=conv1,
110 filters=64,
111 kernel_size=[5, 5],
112 strides=(2, 2),
113 padding="same",
114 activation=tf.nn.relu)
115
116 conv3 = tf.layers.conv2d(
117 inputs=conv2,
118 filters=64,
119 kernel_size=[5, 5],
120 strides=(2, 2),
121 padding="same",
122 activation=tf.nn.relu)
123
124 conv4 = tf.layers.conv2d(
125 inputs=conv3,
126 filters=64,
127 kernel_size=[5, 5],
128 strides=(2, 2),
129 padding="same",
130 activation=tf.nn.relu)
131
132 conv5 = tf.layers.conv2d(
133 inputs=conv4,
134 filters=64,
135 kernel_size=[5, 5],
136 strides=(2, 2),
137 padding="same",
138 activation=tf.nn.relu)
139
140 conv5_flat = tf.layers.flatten(
141 inputs=conv5)
142
143 dense6 = tf.layers.dense(
144 inputs=conv5_flat,
145 units=512,
146 activation=tf.nn.relu)
147
148 return dense6
149
150 def concatenator(global_x):
151 with tf.variable_scope('C', reuse=tf.AUTO_REUSE):
152 dense1 = tf.layers.dense(
153 inputs=global_x,
154 units=1,
155 activation=tf.sigmoid)
156
157 return dense1
+4
-0
src/.ipynb_checkpoints/run-checkpoint.sh less more
0 #!/usr/bin/env bash
1
2 # Runs train.py and saves the console output to output/out
3 stdbuf -i0 -o0 -e0 python -u train.py | tee output/out
+4
-0
src/.ipynb_checkpoints/run_ld-checkpoint.sh less more
0 #!/usr/bin/env bash
1
2 # Runs train_ld.py and saves the console output to output/out
3 stdbuf -i0 -o0 -e0 python -u train_ld.py | tee output/out
+36
-0
src/.ipynb_checkpoints/test-checkpoint.py less more
0 # proj: image-outpainting
1 # file: test.py
2 # authors: Mark Sabini, Gili Rusak
3 # desc: Script for simulating the training pipeline. Masks out
4 # the sides of an image, feeds it through the network, and
5 # compares the network output to the original image.
6 # -------------------------------------------------------------
7 import tensorflow.compat.v1 as tf
8 tf.disable_v2_behavior()
9 import numpy as np
10 from PIL import Image
11 import model
12 import util
13 import os
14 import sys
15
16 model_PATH='/home/jovyan/work/src/output/models/model2000.ckpt'
17 in_PATH='/home/jovyan/work/images/test.png'
18 out_PATH='/home/jovyan/work/results/test_output.png'
19
20 tf.reset_default_graph()
21
22 IMAGE_SZ = 128
23
24 img = np.array(Image.open(in_PATH).convert('RGB'))[np.newaxis] / 255.0
25 img_p = util.preprocess_images_outpainting(img)
26
27 G_Z = tf.placeholder(tf.float32, shape=[None, IMAGE_SZ, IMAGE_SZ, 4], name='G_Z')
28 G_sample = model.generator(G_Z)
29
30 saver = tf.train.Saver()
31
32 with tf.Session() as sess:
33 saver.restore(sess, model_PATH)
34 output, = sess.run([G_sample], feed_dict={G_Z: img_p})
35 util.save_image(output[0], out_PATH)
+202
-0
src/.ipynb_checkpoints/train-checkpoint.py less more
0 # proj: image-outpainting
1 # file: train.py
2 # authors: Mark Sabini, Gili Rusak
3 # desc: Train the model specified in model.py, which only
4 # uses a global discriminator.
5 # -------------------------------------------------------------
6 import tensorflow.compat.v1 as tf
7 import numpy as np
8 from PIL import Image
9 import model
10 import util
11 import os
12 import sys
13 from tensorflow.python.framework import ops
14 tf.disable_eager_execution()
15 ops.reset_default_graph()
16
17 # Places365 Training Hyperparameters
18 BATCH_SZ = 16
19 VERBOSE = False
20 EPSILON = 1e-9
21 IMAGE_SZ = 128
22 OUT_DIR = 'output'
23 MODEL_DIR = os.path.join(OUT_DIR, 'models')
24 INFO_PATH = os.path.join(OUT_DIR, 'run.txt')
25 N_TEST = 10
26 N_ITERS = 227500
27 N_ITERS_P1 = 40950 # How many iterations to train in phase 1
28 N_ITERS_P2 = 4550 # How many iterations to train in phase 2
29 INTV_PRINT = 200 # How often to print
30 INTV_SAVE = 1000 # How often to save the model
31 ALPHA = 0.0004
32
33 '''
34 # City Training Hyperparameters
35 BATCH_SZ = 1
36 VERBOSE = False
37 EPSILON = 1e-9
38 IMAGE_SZ = 128
39 OUT_DIR = 'output'
40 MODEL_DIR = os.path.join(OUT_DIR, 'models')
41 INFO_PATH = os.path.join(OUT_DIR, 'run.txt')
42 N_TEST = 1
43 N_ITERS = 5000
44 N_ITERS_P1 = 1000 # How many iterations to train in phase 1
45 N_ITERS_P2 = 400 # How many iterations to train in phase 2
46 INTV_PRINT = 50 # How often to print
47 INTV_SAVE = 10000 # How often to save the model
48 ALPHA = 0.0004
49 '''
50
51 # Check that we don't clobber a pre-existing run
52 if len(sys.argv) < 2 and os.path.isdir(OUT_DIR) and len(os.listdir(OUT_DIR)) > 2:
53 print('Warning, OUT_DIR already exists. Aborting.')
54 exit()
55
56 # Load in a model if specified as the second argument.
57 start_iter = 0
58 model_filename = None
59 if len(sys.argv) >= 2:
60 start_iter = int(sys.argv[1])
61 model_filename = os.path.join(MODEL_DIR, 'model%d.ckpt' % start_iter)
62
63
64
65 # Generator code
66 G_Z = tf.placeholder(tf.float32, shape=[None, IMAGE_SZ, IMAGE_SZ, 4], name='G_Z')
67 DG_X = tf.placeholder(tf.float32, shape=[None, IMAGE_SZ, IMAGE_SZ, 3], name='DG_X')
68
69 # Load Places365 data
70 data = np.load('places/places_128.npz')
71 imgs = data['imgs_train'] # Originally from http://data.csail.mit.edu/places/places365/val_256.tar
72 imgs_p = util.preprocess_images_outpainting(imgs)
73
74 test_imgs = data['imgs_test']
75 test_imgs_p = util.preprocess_images_outpainting(test_imgs)
76
77 test_img = test_imgs[:N_TEST]
78 test_img_p = test_imgs_p[:N_TEST]
79
80 train_img = imgs[4, np.newaxis]
81 train_img_p = imgs_p[4, np.newaxis]
82
83 '''
84 # Load city image data
85 imgs = util.load_city_image()
86 imgs_p = util.preprocess_images_outpainting(imgs)
87
88 test_imgs = util.load_city_image()
89 test_imgs_p = util.preprocess_images_outpainting(test_imgs)
90
91 test_img = test_imgs
92 test_img_p = test_imgs_p
93
94 train_img = imgs
95 train_img_p = imgs_p
96 '''
97
98 # Write training and testing sample ground truths as reference
99 util.save_image(train_img[0], os.path.join(OUT_DIR, 'train_img.png'))
100 for i_test in range(N_TEST):
101 util.save_image(test_imgs[i_test], os.path.join(OUT_DIR, 'test_img_%d.png' % i_test))
102
103 G_sample = model.generator(G_Z)
104 vars_G = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='G')
105
106 C_real = model.concatenator(model.global_discriminator(DG_X))
107 C_fake = model.concatenator(model.global_discriminator(G_sample))
108 vars_DG = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='DG')
109 vars_C = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='C')
110
111 C_loss = -tf.reduce_mean(tf.log(tf.maximum(C_real, EPSILON)) + tf.log(tf.maximum(1. - C_fake, EPSILON)))
112 G_MSE_loss = tf.losses.mean_squared_error(G_sample, DG_X, weights=tf.expand_dims(G_Z[:,:,:,3], -1)) # TODO: MULTIPLY with mask. Actually see if we want to remove this.
113 G_loss = G_MSE_loss - ALPHA * tf.reduce_mean(tf.log(tf.maximum(C_fake, EPSILON)))
114
115 C_solver = tf.train.AdamOptimizer().minimize(C_loss, var_list=(vars_DG + vars_C))
116 G_solver = tf.train.AdamOptimizer().minimize(G_loss, var_list=vars_G)
117 G_MSE_solver = tf.train.AdamOptimizer().minimize(G_MSE_loss, var_list=vars_G)
118
119 train_MSE_loss = []
120 dev_MSE_loss = []
121
122 last_output_PATH = [None] * N_TEST
123
124 assert N_ITERS > N_ITERS_P1 + N_ITERS_P2
125
126 # Saver to save the session
127 saver = tf.train.Saver()
128
129 with tf.Session() as sess:
130 if model_filename is None:
131 sess.run(tf.global_variables_initializer())
132 else:
133 saver.restore(sess, model_filename)
134 for i in range(start_iter, N_ITERS + 1):
135 batch, batch_p = util.sample_random_minibatch(imgs, imgs_p, BATCH_SZ)
136 G_sample_ = None
137 C_loss_curr, G_loss_curr, G_MSE_loss_curr = None, None, None
138 if i < N_ITERS_P1: # Stage 1 - Train Generator Only
139 if i == 0:
140 print('------------------> Beginning Phase 1...')
141 _, G_MSE_loss_curr, G_sample_ = sess.run([G_MSE_solver, G_MSE_loss, G_sample], feed_dict={DG_X: batch, G_Z: batch_p})
142 elif i < N_ITERS_P1 + N_ITERS_P2: # Stage 2 - Train Discriminator Only
143 if i == N_ITERS_P1:
144 print('------------------> Beginning Phase 2...')
145 _, C_loss_curr, C_real_, C_fake_ = sess.run([C_solver, C_loss, C_real, C_fake], feed_dict={DG_X: batch, G_Z: batch_p})
146 if VERBOSE:
147 print((i, C_loss_curr, np.min(C_real_), np.max(C_real_), np.min(C_fake_), np.max(C_fake_)))
148 else: # Stage 3 - Train both Generator and Discriminator
149 if i == N_ITERS_P1 + N_ITERS_P2:
150 print('------------------> Beginning Phase 3...')
151 _, C_loss_curr, C_real_, C_fake_ = sess.run([C_solver, C_loss, C_real, C_fake], feed_dict={DG_X: batch, G_Z: batch_p})
152 if VERBOSE:
153 print((i, C_loss_curr, 'D', np.min(C_real_), np.max(C_real_), np.min(C_fake_), np.max(C_fake_)))
154 _, G_loss_curr, G_MSE_loss_curr, G_sample_, C_fake_ = sess.run([G_solver, G_loss, G_MSE_loss, G_sample, C_fake], feed_dict={DG_X: batch, G_Z: batch_p})
155 if VERBOSE:
156 print((i, G_loss_curr, 'G', np.min(C_fake_), np.max(C_fake_)))
157
158 # Periodically test the generator on held-out images
159 if i % INTV_PRINT == 0:
160 G_MSE_loss_curr_dev = None
161 if G_sample_ is not None:
162 # Print out the dev image
163 output, G_MSE_loss_curr_dev = sess.run([G_sample, G_MSE_loss], feed_dict={DG_X: test_img, G_Z: test_img_p})
164 for i_test in range(N_TEST):
165 util.save_image(output[i_test], os.path.join(OUT_DIR, 'dev_%d_%d.png' % (i_test, i)))
166 last_output_PATH[i_test] = os.path.join(OUT_DIR, 'dev_%d_%d.png' % (i_test, i))
167 # Also save the train image
168 output, = sess.run([G_sample], feed_dict={DG_X: train_img, G_Z: train_img_p})
169 util.save_image(output[0], os.path.join(OUT_DIR, 'train%d.png' % i))
170 print('Iteration [%d/%d]:' % (i, N_ITERS))
171 if G_MSE_loss_curr is not None:
172 print('\tG_MSE_loss (train) = %f' % G_MSE_loss_curr)
173 if G_MSE_loss_curr_dev is not None:
174 print('\tG_MSE_loss (dev) = %f' % G_MSE_loss_curr_dev)
175 if G_loss_curr is not None:
176 print('\tG_loss = %f' % G_loss_curr)
177 if C_loss_curr is not None:
178 print('\tC_loss = %f' % C_loss_curr)
179
180 # Keep track of losses for logging
181 if G_MSE_loss_curr is not None:
182 train_MSE_loss.append([i, G_MSE_loss_curr])
183 if G_MSE_loss_curr_dev is not None:
184 dev_MSE_loss.append([i, G_MSE_loss_curr_dev])
185
186 # Save the model every so often
187 if i % INTV_SAVE == 0:
188 save_path = saver.save(sess, os.path.join(MODEL_DIR, 'model%d.ckpt' % i))
189 print('Model saved in path: %s' % save_path)
190
191 # Save the loss every so often
192 if i % INTV_SAVE == 0:
193 np.savez(os.path.join(OUT_DIR, 'loss.npz'), train_MSE_loss=np.array(train_MSE_loss), dev_MSE_loss=np.array(dev_MSE_loss))
194
195 # Save the loss
196 np.savez(os.path.join(OUT_DIR, 'loss.npz'), train_MSE_loss=np.array(train_MSE_loss), dev_MSE_loss=np.array(dev_MSE_loss))
197 # Save the final blended output, and make a graph of the loss.
198 util.plot_loss(os.path.join(OUT_DIR, 'loss.npz'), 'MSE Loss During Training', os.path.join(OUT_DIR, 'loss_plot.png'))
199 for i_test in range(N_TEST):
200 util.postprocess_images_outpainting(os.path.join(OUT_DIR, 'test_img_%d.png' % i_test), last_output_PATH[i_test], os.path.join(OUT_DIR, 'out_paste_%d.png' % i_test), blend=False)
201 util.postprocess_images_outpainting(os.path.join(OUT_DIR, 'test_img_%d.png' % i_test), last_output_PATH[i_test], os.path.join(OUT_DIR, 'out_blend_%d.png' % i_test), blend=True)
+200
-0
src/.ipynb_checkpoints/train_ld-checkpoint.py less more
0 # proj: image-outpainting
1 # file: train_ld.py
2 # authors: Mark Sabini, Gili Rusak
3 # desc: Train the model specified in model_ld.py, which
4 # uses both global and local discriminators.
5 # -------------------------------------------------------------
6 import tensorflow as tf
7 import numpy as np
8 from PIL import Image
9 import model_ld as model
10 import util
11 import os
12 import sys
13
14 tf.reset_default_graph()
15
16 # Places365 Training Hyperparameters
17 BATCH_SZ = 16
18 VERBOSE = False
19 EPSILON = 1e-9
20 IMAGE_SZ = 128
21 OUT_DIR = 'output'
22 MODEL_DIR = os.path.join(OUT_DIR, 'models')
23 INFO_PATH = os.path.join(OUT_DIR, 'run.txt')
24 N_TEST = 10
25 N_ITERS = 64000
26 N_ITERS_P1 = 20000 # How many iterations to train in phase 1
27 N_ITERS_P2 = 4000 # How many iterations to train in phase 2
28 INTV_PRINT = 200 # How often to print
29 INTV_SAVE = 1000 # How often to save the model
30 ALPHA = 0.0004
31
32 '''
33 # City Training Hyperparameters
34 BATCH_SZ = 1
35 VERBOSE = False
36 EPSILON = 1e-9
37 IMAGE_SZ = 128
38 OUT_DIR = 'output'
39 MODEL_DIR = os.path.join(OUT_DIR, 'models')
40 INFO_PATH = os.path.join(OUT_DIR, 'run.txt')
41 N_TEST = 1
42 N_ITERS = 5000
43 N_ITERS_P1 = 1000 # How many iterations to train in phase 1
44 N_ITERS_P2 = 400 # How many iterations to train in phase 2
45 INTV_PRINT = 50 # How often to print
46 INTV_SAVE = 10000 # How often to save the model
47 ALPHA = 0.0004
48 '''
49
50 # Check that we don't clobber a pre-existing run
51 if len(sys.argv) < 2 and os.path.isdir(OUT_DIR) and len(os.listdir(OUT_DIR)) > 2:
52 print('Warning, OUT_DIR already exists. Aborting.')
53 exit()
54
55 # Load in a model if specified as the second argument.
56 start_iter = 0
57 model_filename = None
58 if len(sys.argv) >= 2:
59 start_iter = int(sys.argv[1])
60 model_filename = os.path.join(MODEL_DIR, 'model%d.ckpt' % start_iter)
61
62 # Generator code
63 G_Z = tf.placeholder(tf.float32, shape=[None, IMAGE_SZ, IMAGE_SZ, 4], name='G_Z')
64 DG_X = tf.placeholder(tf.float32, shape=[None, IMAGE_SZ, IMAGE_SZ, 3], name='DG_X')
65
66 # Load Places365 data
67 data = np.load('places/places_128.npz')
68 imgs = data['imgs_train'] # Originally from http://data.csail.mit.edu/places/places365/val_256.tar
69 imgs_p = util.preprocess_images_outpainting(imgs)
70
71 test_imgs = data['imgs_test']
72 test_imgs_p = util.preprocess_images_outpainting(test_imgs)
73
74 test_img = test_imgs[:N_TEST]
75 test_img_p = test_imgs_p[:N_TEST]
76
77 train_img = imgs[4, np.newaxis]
78 train_img_p = imgs_p[4, np.newaxis]
79
80 '''
81 # Load city image data
82 imgs = util.load_city_image()
83 imgs_p = util.preprocess_images_outpainting(imgs)
84
85 test_imgs = util.load_city_image()
86 test_imgs_p = util.preprocess_images_outpainting(test_imgs)
87
88 test_img = test_imgs
89 test_img_p = test_imgs_p
90
91 train_img = imgs
92 train_img_p = imgs_p
93 '''
94
95 # Write training and testing sample ground truths as reference
96 util.save_image(train_img[0], os.path.join(OUT_DIR, 'train_img.png'))
97 for i_test in range(N_TEST):
98 util.save_image(test_imgs[i_test], os.path.join(OUT_DIR, 'test_img_%d.png' % i_test))
99
100 G_sample = model.generator(G_Z)
101 vars_G = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='G')
102
103 C_real = model.concatenator(model.global_discriminator(DG_X), model.local_discriminator(DG_X[:, :, :IMAGE_SZ // 2, :]), model.local_discriminator(tf.reverse(DG_X[:, :, -IMAGE_SZ // 2:, :], axis=[2])))
104 C_fake = model.concatenator(model.global_discriminator(G_sample), model.local_discriminator(G_sample[:, :, :IMAGE_SZ // 2, :]), model.local_discriminator(tf.reverse(G_sample[:, :, -IMAGE_SZ // 2:, :], axis=[2])))
105 vars_DG = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='DG')
106 vars_DL = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='DL')
107 vars_C = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope='C')
108
109 C_loss = -tf.reduce_mean(tf.log(tf.maximum(C_real, EPSILON)) + tf.log(tf.maximum(1. - C_fake, EPSILON)))
110 G_MSE_loss = tf.losses.mean_squared_error(G_sample, DG_X, weights=tf.expand_dims(G_Z[:,:,:,3], -1)) # TODO: MULTIPLY with mask. Actually see if we want to remove this.
111 G_loss = G_MSE_loss - ALPHA * tf.reduce_mean(tf.log(tf.maximum(C_fake, EPSILON)))
112
113 C_solver = tf.train.AdamOptimizer().minimize(C_loss, var_list=(vars_DG + vars_DL + vars_C))
114 G_solver = tf.train.AdamOptimizer().minimize(G_loss, var_list=vars_G)
115 G_MSE_solver = tf.train.AdamOptimizer().minimize(G_MSE_loss, var_list=vars_G)
116
117 train_MSE_loss = []
118 dev_MSE_loss = []
119
120 last_output_PATH = [None] * N_TEST
121
122 assert N_ITERS > N_ITERS_P1 + N_ITERS_P2
123
124 # Saver to save the session
125 saver = tf.train.Saver()
126
127 with tf.Session() as sess:
128 if model_filename is None:
129 sess.run(tf.global_variables_initializer())
130 else:
131 saver.restore(sess, model_filename)
132 for i in range(start_iter, N_ITERS + 1):
133 batch, batch_p = util.sample_random_minibatch(imgs, imgs_p, BATCH_SZ)
134 G_sample_ = None
135 C_loss_curr, G_loss_curr, G_MSE_loss_curr = None, None, None
136 if i < N_ITERS_P1: # Stage 1 - Train Generator Only
137 if i == 0:
138 print('------------------> Beginning Phase 1...')
139 _, G_MSE_loss_curr, G_sample_ = sess.run([G_MSE_solver, G_MSE_loss, G_sample], feed_dict={DG_X: batch, G_Z: batch_p})
140 elif i < N_ITERS_P1 + N_ITERS_P2: # Stage 2 - Train Discriminator Only
141 if i == N_ITERS_P1:
142 print('------------------> Beginning Phase 2...')
143 _, C_loss_curr, C_real_, C_fake_ = sess.run([C_solver, C_loss, C_real, C_fake], feed_dict={DG_X: batch, G_Z: batch_p})
144 if VERBOSE:
145 print((i, C_loss_curr, np.min(C_real_), np.max(C_real_), np.min(C_fake_), np.max(C_fake_)))
146 else: # Stage 3 - Train both Generator and Discriminator
147 if i == N_ITERS_P1 + N_ITERS_P2:
148 print('------------------> Beginning Phase 3...')
149 _, C_loss_curr, C_real_, C_fake_ = sess.run([C_solver, C_loss, C_real, C_fake], feed_dict={DG_X: batch, G_Z: batch_p})
150 if VERBOSE:
151 print((i, C_loss_curr, 'D', np.min(C_real_), np.max(C_real_), np.min(C_fake_), np.max(C_fake_)))
152 _, G_loss_curr, G_MSE_loss_curr, G_sample_, C_fake_ = sess.run([G_solver, G_loss, G_MSE_loss, G_sample, C_fake], feed_dict={DG_X: batch, G_Z: batch_p})
153 if VERBOSE:
154 print((i, G_loss_curr, 'G', np.min(C_fake_), np.max(C_fake_)))
155
156 # Periodically test the generator on held-out images
157 if i % INTV_PRINT == 0:
158 G_MSE_loss_curr_dev = None
159 if G_sample_ is not None:
160 # Print out the dev image
161 output, G_MSE_loss_curr_dev = sess.run([G_sample, G_MSE_loss], feed_dict={DG_X: test_img, G_Z: test_img_p})
162 for i_test in range(N_TEST):
163 util.save_image(output[i_test], os.path.join(OUT_DIR, 'dev_%d_%d.png' % (i_test, i)))
164 last_output_PATH[i_test] = os.path.join(OUT_DIR, 'dev_%d_%d.png' % (i_test, i))
165 # Also save the train image
166 output, = sess.run([G_sample], feed_dict={DG_X: train_img, G_Z: train_img_p})
167 util.save_image(output[0], os.path.join(OUT_DIR, 'train%d.png' % i))
168 print('Iteration [%d/%d]:' % (i, N_ITERS))
169 if G_MSE_loss_curr is not None:
170 print('\tG_MSE_loss (train) = %f' % G_MSE_loss_curr)
171 if G_MSE_loss_curr_dev is not None:
172 print('\tG_MSE_loss (dev) = %f' % G_MSE_loss_curr_dev)
173 if G_loss_curr is not None:
174 print('\tG_loss = %f' % G_loss_curr)
175 if C_loss_curr is not None:
176 print('\tC_loss = %f' % C_loss_curr)
177
178 # Keep track of losses for logging
179 if G_MSE_loss_curr is not None:
180 train_MSE_loss.append([i, G_MSE_loss_curr])
181 if G_MSE_loss_curr_dev is not None:
182 dev_MSE_loss.append([i, G_MSE_loss_curr_dev])
183
184 # Save the model every so often
185 if i % INTV_SAVE == 0 and i > 0:
186 save_path = saver.save(sess, os.path.join(MODEL_DIR, 'model%d.ckpt' % i))
187 print('Model saved in path: %s' % save_path)
188
189 # Save the loss every so often
190 if i % INTV_SAVE == 0:
191 np.savez(os.path.join(OUT_DIR, 'loss.npz'), train_MSE_loss=np.array(train_MSE_loss), dev_MSE_loss=np.array(dev_MSE_loss))
192
193 # Save the loss
194 np.savez(os.path.join(OUT_DIR, 'loss.npz'), train_MSE_loss=np.array(train_MSE_loss), dev_MSE_loss=np.array(dev_MSE_loss))
195 # Save the final blended output, and make a graph of the loss.
196 util.plot_loss(os.path.join(OUT_DIR, 'loss.npz'), 'MSE Loss During Training', os.path.join(OUT_DIR, 'loss_plot.png'))
197 for i_test in range(N_TEST):
198 util.postprocess_images_outpainting(os.path.join(OUT_DIR, 'test_img_%d.png' % i_test), last_output_PATH[i_test], os.path.join(OUT_DIR, 'out_paste_%d.png' % i_test), blend=False)
199 util.postprocess_images_outpainting(os.path.join(OUT_DIR, 'test_img_%d.png' % i_test), last_output_PATH[i_test], os.path.join(OUT_DIR, 'out_blend_%d.png' % i_test), blend=True)
+274
-0
src/.ipynb_checkpoints/util-checkpoint.py less more
0 # proj: image-outpainting
1 # file: util.py
2 # authors: Mark Sabini, Gili Rusak
3 # desc: Various utility functions for all sorts of things.
4 # -------------------------------------------------------------
5 import numpy as np
6 from PIL import Image
7 import scipy.misc
8 import matplotlib.pyplot as plt
9 import cv2
10 import os
11 import re
12 import imageio
13
14 IMAGE_SZ = 128 # Should be a power of 2
15
16 # Loads the city image.
17 # Returns: normalized numpy array of size (1, IMAGE_SZ, IMAGE_SZ, 3)
18 def load_city_image():
19 im = Image.open('images/city_128.png').convert('RGB')
20 width, height = im.size
21 left = (width - IMAGE_SZ) / 2
22 top = (height - IMAGE_SZ) / 2
23 im = im.crop((left, top, left + IMAGE_SZ, top + IMAGE_SZ))
24 pix = np.array(im)
25 assert pix.shape == (IMAGE_SZ, IMAGE_SZ, 3)
26 return pix[np.newaxis] / 255.0 # Need to normalize images to [0, 1]
27
28 # Loads multiple images from a directory.
29 # Returns: normalized numpy array of size (m, IMAGE_SZ, IMAGE_SZ, 3)
30 def load_images(in_PATH, verbose=False):
31 imgs = []
32 for filename in sorted(os.listdir(in_PATH)):
33 if verbose:
34 print('Processing %s' % filename)
35 full_filename = os.path.join(os.path.abspath(in_PATH), filename)
36 img = Image.open(full_filename).convert('RGB')
37 pix = np.array(img)
38 pix_norm = pix / 255.0
39 imgs.append(pix_norm)
40 return np.array(imgs)
41
42 # Reads in all the images in a directory and saves them to an .npy file.
43 def compile_images(in_PATH, out_PATH):
44 imgs = load_images(in_PATH, verbose=True)
45 np.save(out_PATH, imgs)
46
47 # Masks and preprocesses an (m, IMAGE_SZ, IMAGE_SZ, 3) batch of images for image outpainting.
48 # Returns: numpy array of size (m, IMAGE_SZ, IMAGE_SZ, 4)
49 def preprocess_images_outpainting(imgs, crop=True):
50 m = imgs.shape[0]
51 imgs = np.array(imgs, copy=True)
52 pix_avg = np.mean(imgs, axis=(1, 2, 3))
53 if crop:
54 imgs[:, :, :int(2 * IMAGE_SZ / 8), :] = imgs[:, :, int(-2 * IMAGE_SZ / 8):, :] = pix_avg[:, np.newaxis, np.newaxis, np.newaxis]
55 mask = np.zeros((m, IMAGE_SZ, IMAGE_SZ, 1))
56 mask[:, :, :int(2 * IMAGE_SZ / 8), :] = mask[:, :, int(-2 * IMAGE_SZ / 8):, :] = 1.0
57 imgs_p = np.concatenate((imgs, mask), axis=3)
58 return imgs_p
59
60 # Expands and preprocesses a single (h, w, 3) image for image outpainting.
61 # Returns: numpy array of size (h, w + 2 * dw, 4)
62 def preprocess_images_gen(img):
63 img = np.array(img, copy=True)
64 pix_avg = np.mean(img)
65 dw = int(2 * IMAGE_SZ / 8) # Amount that will be outpainted on each side
66 img_expand = np.ones((img.shape[0], img.shape[1] + 2 * dw, img.shape[2])) * pix_avg
67 img_expand[:, dw:-dw, :] = img
68 mask = np.zeros((img_expand.shape[0], img_expand.shape[1], 1))
69 mask[:, :int(2 * IMAGE_SZ / 8), :] = mask[:, int(-2 * IMAGE_SZ / 8):, :] = 1.0
70 img_p = np.concatenate((img_expand, mask), axis=2)
71 return img_p[np.newaxis]
72
73 # Renormalizes an image to [0, 255].
74 def norm_image(img_r):
75 img_norm = (img_r * 255.0).astype(np.uint8)
76 return img_norm
77
78 # Visualize an image.
79 def vis_image(img_r, mode='RGB'):
80 img_norm = norm_image(img_r)
81 img = Image.fromarray(img_norm, mode)
82 img.show()
83
84 # Save an image as a .png file.
85 def save_image(img_r, name, mode='RGB'):
86 img_norm = norm_image(img_r)
87 img = Image.fromarray(img_norm, mode)
88 img.save(name, format='PNG')
89
90 # Sample a random minibatch from data.
91 # Returns: Two numpy arrays, representing examples and their corresponding
92 # preprocessed arrays.
93 def sample_random_minibatch(data, data_p, m):
94 indices = np.random.randint(0, data.shape[0], m)
95 return data[indices], data_p[indices]
96
97 # Plots the loss and saves the plot.
98 def plot_loss(loss_filename, title, out_filename):
99 loss = np.load(loss_filename)
100 assert 'train_MSE_loss' in loss and 'dev_MSE_loss' in loss
101 train_MSE_loss = loss['train_MSE_loss']
102 dev_MSE_loss = loss['dev_MSE_loss'] # TODO: Deal with dev_MSE_loss not changing during Phase 2
103 label_train, = plt.plot(train_MSE_loss[:, 0], train_MSE_loss[:, 1], label='Training MSE loss')
104 label_dev, = plt.plot(dev_MSE_loss[:, 0], dev_MSE_loss[:, 1], label='Dev MSE loss')
105 plt.legend(handles=[label_train, label_dev])
106 plt.xlabel('Iteration')
107 plt.ylabel('MSE Loss')
108 plt.title(title)
109 plt.savefig(out_filename)
110 plt.clf()
111
112 # Plots the loss and saves the plot, but fancier.
113 def plot_loss2(loss_filename, title, out_filename):
114 loss = np.load(loss_filename)
115 itrain_MSE_loss, train_MSE_loss = loss['itrain_MSE_loss'], loss['train_MSE_loss']
116 idev_MSE_loss, dev_MSE_loss = loss['idev_MSE_loss'], loss['dev_MSE_loss']
117 iG_loss, G_loss = loss['iG_loss'], loss['G_loss']
118 iD_loss, D_loss = loss['iD_loss'], loss['D_loss']
119 label_train, = plt.plot(itrain_MSE_loss, train_MSE_loss, label='Training MSE loss')
120 label_dev, = plt.plot(idev_MSE_loss, dev_MSE_loss, label='Dev MSE loss')
121 label_G, = plt.plot(iG_loss, G_loss, label='Generator loss')
122 label_D, = plt.plot(iD_loss, D_loss, label='Discriminator loss')
123 plt.legend(handles=[label_train, label_dev, label_G, label_D])
124 plt.xlabel('Iteration')
125 plt.ylabel('Loss')
126 plt.title(title)
127 plt.savefig(out_filename)
128 plt.clf()
129
130 # Use seamless cloning to improve the generator's output.
131 def postprocess_images_outpainting(img_PATH, img_o_PATH, out_PATH, blend=False): # img, img_0 are (64, 64, 3), mask is (64, 64, 1)
132 src = cv2.imread(img_PATH)[:, int(2 * IMAGE_SZ / 8):-int(2 * IMAGE_SZ / 8), :]
133 dst = cv2.imread(img_o_PATH)
134 if blend:
135 mask = np.ones(src.shape, src.dtype) * 255
136 center = (int(IMAGE_SZ / 2) - 1, int(IMAGE_SZ / 2) - 1)
137 out = cv2.seamlessClone(src, dst, mask, center, cv2.NORMAL_CLONE)
138 else:
139 out = dst.copy()
140 out[:, int(2 * IMAGE_SZ / 8):-int(2 * IMAGE_SZ / 8), :] = src
141 cv2.imwrite(out_PATH, out)
142
143 # Use seamless cloning to improve the generator's output.
144 def postprocess_images_gen(img, img_o, blend=False):
145 src = img[:, :, ::-1].copy()
146 dst = img_o[:, :, ::-1].copy()
147 if blend:
148 mask = np.ones(src.shape, src.dtype) * 255
149 center = (int(dst.shape[1] / 2) - 1, int(dst.shape[0] / 2) - 1)
150 out = cv2.seamlessClone(src, dst, mask, center, cv2.NORMAL_CLONE)
151 else:
152 out = dst.copy()
153 out[:, int(2 * IMAGE_SZ / 8):-int(2 * IMAGE_SZ / 8), :] = src
154 return out[:, :, ::-1].copy()
155
156 # Crop and resize all the images in a directory.
157 def resize_images(src_PATH, dst_PATH):
158 for filename in os.listdir(src_PATH):
159 print('Processing %s' % filename)
160 full_filename = os.path.join(os.path.abspath(src_PATH), filename)
161 img_raw = Image.open(full_filename).convert('RGB')
162 w, h = img_raw.size
163 if w <= h:
164 dim = w
165 y_start = int((h - dim) / 2)
166 img_crop = img_raw.crop(box=(0, y_start, dim, y_start + dim))
167 else: # w > h
168 dim = h
169 x_start = int((w - dim) / 2)
170 img_crop = img_raw.crop(box=(x_start, 0, x_start + dim, dim))
171 img_scale = img_crop.resize((IMAGE_SZ, IMAGE_SZ), Image.ANTIALIAS)
172 full_outfilename = os.path.join(os.path.abspath(dst_PATH), filename)
173 img_scale.save(full_outfilename, format='PNG')
174
175 # Parse the output of train.py to extract the various losses.
176 def parse_log(in_PATH, out_PATH):
177 data = []
178 curr_list = []
179 with open(in_PATH, 'r') as fp:
180 for i, line in enumerate(fp):
181 if i == 0:
182 continue
183 line = line.strip()
184 if line.startswith('----'):
185 continue
186 elif line.startswith('Model'):
187 continue
188 elif line.startswith('Iteration'):
189 if len(curr_list):
190 data.append(curr_list)
191 curr_list = []
192 curr_list.append(line)
193 else:
194 curr_list.append(line)
195 if len(curr_list):
196 data.append(curr_list)
197 G_MSE_train, G_MSE_dev, G, C = None, None, None, None
198 G_MSE_train_s, G_MSE_dev_s, G_s, C_s = [], [], [], []
199 G_MSE_train_is, G_MSE_dev_is, G_is, C_is = [], [], [], []
200 def extract_loss(str):
201 return float(re.findall('= ([\d, .]+)', str)[0])
202 for entry in data:
203 i = int(re.findall('\[(\d+)/', entry[0])[0])
204 if len(entry) == 3: # Phase 1
205 G_MSE_train = extract_loss(entry[1])
206 G_MSE_dev = extract_loss(entry[2])
207 elif len(entry) == 2: # Phase 2
208 C = extract_loss(entry[1])
209 elif len(entry) == 5: # Phase 3
210 G_MSE_train = extract_loss(entry[1])
211 G_MSE_dev = extract_loss(entry[2])
212 G = extract_loss(entry[3])
213 C = extract_loss(entry[4])
214 if G_MSE_train is not None:
215 G_MSE_train_s.append(G_MSE_train)
216 G_MSE_train_is.append(i)
217 if G_MSE_dev is not None:
218 G_MSE_dev_s.append(G_MSE_dev)
219 G_MSE_dev_is.append(i)
220 if G is not None:
221 G_s.append(G)
222 G_is.append(i)
223 if C is not None:
224 C_s.append(C)
225 C_is.append(i)
226 G_MSE_train_sm = np.array(G_MSE_train_s)
227 G_MSE_dev_sm = np.array(G_MSE_dev_s)
228 G_sm = np.array(G_s)
229 C_sm = np.array(C_s)
230 G_MSE_train_ism = np.array(G_MSE_train_is)
231 G_MSE_dev_ism = np.array(G_MSE_dev_is)
232 G_ism = np.array(G_is)
233 C_ism = np.array(C_is)
234 np.savez(out_PATH, train_MSE_loss=G_MSE_train_sm, dev_MSE_loss=G_MSE_dev_sm, G_loss=G_sm, D_loss=C_sm,
235 itrain_MSE_loss=G_MSE_train_ism, idev_MSE_loss=G_MSE_dev_ism, iG_loss=G_ism, iD_loss=C_ism)
236
237 # Smoothes the MSE loss in the output loss file to make plotting easier.
238 def smooth_MSE_loss(loss_file, window_size, outfile):
239 losses = np.load(loss_file)
240 train = losses['train_MSE_loss']
241 dev = losses['dev_MSE_loss']
242 num_train = train.shape[0]
243 new_train_list = []
244 for i in range(0, num_train, window_size):
245 window_avg = np.sum(train[i:i+window_size, 1]) / float(window_size)
246 window_avg_val = np.sum(train[i:i+window_size, 0]) / float(window_size)
247 new_train_list.append([window_avg_val, window_avg])
248 np_train = np.array(new_train_list[:-2])
249 np.savez(outfile, train_MSE_loss=np_train, dev_MSE_loss=dev)
250
251 # Create a GIF to enable visualization of generator outputs over the course of training.
252 def create_GIF(in_PATH, prefix, out_PATH):
253 indices = range(0, 227401, 200)
254 images = []
255 for index in indices:
256 full_filename = os.path.join(os.path.abspath(in_PATH), prefix + str(index) + '.png')
257 try:
258 images.append(imageio.imread(full_filename))
259 except:
260 continue
261 images = images[:50] + images[50::10] + [images[-1]]
262 imageio.mimwrite(out_PATH, images, loop=1, duration=0.1)
263
264 # Compute the RMSE between a ground truth and outpainted image.
265 def compute_RMSE(image_gt_PATH, image_o_PATH):
266 im_gt = np.array(Image.open(image_gt_PATH).convert('RGB')).astype(np.float64)
267 im_o = np.array(Image.open(image_o_PATH).convert('RGB')).astype(np.float64)
268 assert im_gt.shape == (128, 128, 3)
269 assert im_o.shape == (128, 128, 3)
270 M = np.ones((128, 128, 3))
271 M[:, 32:96, :] = 0
272 num_pixels = 128 * 64 * 3
273 return np.sqrt(np.sum(((im_gt - im_o) * M) ** 2) / num_pixels)
+130
-0
src/Untitled.ipynb less more
0 {
1 "cells": [
2 {
3 "cell_type": "code",
4 "execution_count": 3,
5 "id": "ea42a489",
6 "metadata": {},
7 "outputs": [
8 {
9 "name": "stderr",
10 "output_type": "stream",
11 "text": [
12 "2022-06-27 17:55:39.372674: W tensorflow/stream_executor/platform/default/dso_loader.cc:59] Could not load dynamic library 'libcudart.so.10.1'; dlerror: libcudart.so.10.1: cannot open shared object file: No such file or directory\n",
13 "2022-06-27 17:55:39.372709: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n"
14 ]
15 },
16 {
17 "name": "stdout",
18 "output_type": "stream",
19 "text": [
20 "WARNING:tensorflow:From /usr/local/lib/python3.7/dist-packages/tensorflow/python/compat/v2_compat.py:96: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version.\n",
21 "Instructions for updating:\n",
22 "non-resource variables are not supported in the long term\n",
23 "Imported model (for Places365, 128x128 images)\n"
24 ]
25 }
26 ],
27 "source": [
28 "import tensorflow.compat.v1 as tf\n",
29 "tf.disable_v2_behavior()\n",
30 "tf.reset_default_graph()\n",
31 "import numpy as np\n",
32 "from PIL import Image\n",
33 "import model\n",
34 "import util\n",
35 "import os\n",
36 "import sys"
37 ]
38 },
39 {
40 "cell_type": "code",
41 "execution_count": 4,
42 "id": "24a28be9",
43 "metadata": {},
44 "outputs": [],
45 "source": [
46 "model_PATH='/home/jovyan/work/src/output/models/model2000.ckpt'"
47 ]
48 },
49 {
50 "cell_type": "code",
51 "execution_count": 8,
52 "id": "2f04ed3d",
53 "metadata": {},
54 "outputs": [],
55 "source": [
56 "def load_demo_image(in_PATH):\n",
57 " img = np.array(Image.open(in_PATH).convert('RGB'))[np.newaxis] / 255.0\n",
58 " img_p = util.preprocess_images_outpainting(img)\n",
59 " return img_p"
60 ]
61 },
62 {
63 "cell_type": "code",
64 "execution_count": 6,
65 "id": "4cc717e4",
66 "metadata": {},
67 "outputs": [],
68 "source": [
69 "def inference(model_PATH, img_p):\n",
70 " G_Z = tf.placeholder(tf.float32, shape=[None, IMAGE_SZ, IMAGE_SZ, 4], name='G_Z')\n",
71 " G_sample = model.generator(G_Z)\n",
72 " \n",
73 " saver = tf.train.Saver()\n",
74 " with tf.Session() as sess:\n",
75 " saver.restore(sess, model_PATH)\n",
76 " output, = sess.run([G_sample], feed_dict={G_Z: img_p})\n",
77 " img_norm = (output[0] * 255.0).astype(np.uint8)\n",
78 " img = Image.fromarray(img_norm, 'RGB')\n",
79 " #util.save_image(output[0], out_PATH)\n",
80 " return img"
81 ]
82 },
83 {
84 "cell_type": "code",
85 "execution_count": null,
86 "id": "9ce2fc25",
87 "metadata": {},
88 "outputs": [],
89 "source": [
90 "def handle(conf):\n",
91 " \"\"\"\n",
92 " 该方法是部署之后，其他人调用你的服务时候的处理方法。\n",
93 " 请按规范填写参数结构，这样我们就能替你自动生成配置文件，方便其他人的调用。\n",
94 " 范例：\n",
95 " params['key'] = value # value_type: str # description: some description\n",
96 " value_type 可以选择：img, video, audio, str, int, float, [int], [str], [float]\n",
97 " 参数请放到params字典中，我们会自动解析该变量。\n",
98 " \"\"\"\n",
99 " base64_str = conf['Photo']\n",
100 " image = load_demo_image(base64_str, image_size, device)\n",
101 " res = inference(model, image)\n",
102 " # add your code\n",
103 " return {'Output': res}\n",
104 " "
105 ]
106 }
107 ],
108 "metadata": {
109 "kernelspec": {
110 "display_name": "Python 3 (ipykernel)",
111 "language": "python",
112 "name": "python3"
113 },
114 "language_info": {
115 "codemirror_mode": {
116 "name": "ipython",
117 "version": 3
118 },
119 "file_extension": ".py",
120 "mimetype": "text/x-python",
121 "name": "python",
122 "nbconvert_exporter": "python",
123 "pygments_lexer": "ipython3",
124 "version": "3.7.5"
125 }
126 },
127 "nbformat": 4,
128 "nbformat_minor": 5
129 }
+2
-1
src/model.py less more
33 # desc: Model for outpainting on 128x128 images with only
44 # a global discriminator.
55 # -------------------------------------------------------------
6 import tensorflow as tf
6 import tensorflow.compat.v1 as tf
7 tf.disable_v2_behavior()
78
89 print('Imported model (for Places365, 128x128 images)')
910
+2
-0
src/output/models/checkpoint less more
0 model_checkpoint_path: "model0.ckpt"
1 all_model_checkpoint_paths: "model0.ckpt"
src/output/models/model0.ckpt.data-00000-of-00001 less more
Binary diff not shown
src/output/models/model0.ckpt.index less more
Binary diff not shown
src/output/models/model0.ckpt.meta less more
Binary diff not shown
src/output/models/model2000.ckpt.data-00000-of-00001 less more
Binary diff not shown
src/output/models/model2000.ckpt.index less more
Binary diff not shown
src/output/models/model2000.ckpt.meta less more
Binary diff not shown
+5
-6
src/test.py less more
44 # the sides of an image, feeds it through the network, and
55 # compares the network output to the original image.
66 # -------------------------------------------------------------
7 import tensorflow as tf
7 import tensorflow.compat.v1 as tf
8 tf.disable_v2_behavior()
89 import numpy as np
910 from PIL import Image
1011 import model
1213 import os
1314 import sys
1415
15 if len(sys.argv) != 4:
16 print('Usage: python test.py [model_PATH] [in_PATH] [out_PATH]')
17 exit()
18
19 _, model_PATH, in_PATH, out_PATH = sys.argv
16 model_PATH='/home/jovyan/work/src/output/models/model2000.ckpt'
17 in_PATH='/home/jovyan/work/images/test.png'
18 out_PATH='/home/jovyan/work/results/test_output.png'
2019
2120 tf.reset_default_graph()
2221
+6
-3
src/train.py less more
33 # desc: Train the model specified in model.py, which only
44 # uses a global discriminator.
55 # -------------------------------------------------------------
6 import tensorflow as tf
6 import tensorflow.compat.v1 as tf
77 import numpy as np
88 from PIL import Image
99 import model
1010 import util
1111 import os
1212 import sys
13
14 tf.reset_default_graph()
13 from tensorflow.python.framework import ops
14 tf.disable_eager_execution()
15 ops.reset_default_graph()
1516
1617 # Places365 Training Hyperparameters
1718 BATCH_SZ = 16
5859 if len(sys.argv) >= 2:
5960 start_iter = int(sys.argv[1])
6061 model_filename = os.path.join(MODEL_DIR, 'model%d.ckpt' % start_iter)
62
63
6164
6265 # Generator code
6366 G_Z = tf.placeholder(tf.float32, shape=[None, IMAGE_SZ, IMAGE_SZ, 4], name='G_Z')