我试图预处理我的图像，将它们添加到4D阵列。它从正确开始，但它随着时间的推移变慢，我认为这是由于我的CPU，但我尝试在云上的GPU上运行它，它仍然会变慢。这是由于RAM吗？如何优化它以更快地运行？
import tensorflow as tf
import os
import glob
import numpy as np
from PIL import Image
from random import randint
sess = tf.InteractiveSession()
def process_image(filename):
im = Image.open(filename)
array = np.array(im,dtype=np.uint8)
#Resize and normalize
resized = tf.image.resize_images(array, size[0], size[1], method = 0)
normalized = tf.image.per_image_whitening(resized)
result = sess.run(normalized)
return result
counter_train = 0
counter_val = 0
for i, foldername in enumerate(foldernames):
ind = 0
index = randint(ind,ind+29)
for j, filename in enumerate(glob.glob(foldername + '*.ppm')):
print filename
result = process_image(filename)
if j == index:
npX_val[counter_val]=result
npClass_val[counter_val]=i
ind += 30
index = randint(ind,ind+29)
counter_val += 1
else:
npX_train[counter_train]=result
npClass_train[counter_train]=i
counter_train += 1
print counter_val
print counter_train
我也运行了pyinstrument，我得到了这个
3.160 <module> process.py:1
└─ 2.763 <module> tensorflow/__init__.py:19
└─ 2.761 <module> tensorflow/python/__init__.py:26
├─ 2.144 <module> tensorflow/contrib/__init__.py:15
│ ├─ 0.955 <module> tensorflow/contrib/learn/__init__.py:65
│ │ └─ 0.953 <module> tensorflow/contrib/learn/python/__init__.py:16
│ │ └─ 0.950 <module> tensorflow/contrib/learn/python/learn/__init__.py:16
│ │ ├─ 0.889 <module> tensorflow/contrib/learn/python/learn/estimators/__init__.py:16
│ │ │ ├─ 0.789 <module> tensorflow/contrib/learn/python/learn/estimators/autoencoder.py:16
│ │ │ │ └─ 0.770 <module> tensorflow/contrib/learn/python/learn/estimators/base.py:16
│ │ │ │ └─ 0.764 <module> tensorflow/contrib/learn/python/learn/estimators/estimator.py:16
│ │ │ │ └─ 0.729 <module> tensorflow/contrib/learn/python/learn/learn_io/__init__.py:16
│ │ │ │ └─ 0.724 <module> tensorflow/contrib/learn/python/learn/learn_io/pandas_io.py:16
│ │ │ │ └─ 0.724 <module> pandas/__init__.py:5
│ │ │ │ ├─ 0.307 <module> pandas/core/api.py:5
│ │ │ │ │ └─ 0.283 <module> pandas/core/groupby.py:1
│ │ │ │ │ └─ 0.268 <module> pandas/core/frame.py:10
│ │ │ │ │ ├─ 0.135 <module> pandas/core/series.py:3
│ │ │ │ │ │ └─ 0.116 <module> pandas/tools/plotting.py:3
│ │ │ │ │ │ └─ 0.112 <module> pandas/tseries/converter.py:1
│ │ │ │ │ │ ├─ 0.061 <module> matplotlib/__init__.py:101
│ │ │ │ │ │ └─ 0.044 <module> matplotlib/dates.py:111
│ │ │ │ │ └─ 0.102 <module> pandas/core/generic.py:2
│ │ │ │ │ └─ 0.085 <module> pandas/core/internals.py:1
│ │ │ │ │ └─ 0.075 <module> pandas/sparse/array.py:3
│ │ │ │ │ └─ 0.070 <module> pandas/core/ops.py:5
│ │ │ │ │ └─ 0.066 <module> pandas/computation/__init__.py:2
│ │ │ │ │ └─ 0.065 <module> numexpr/__init__.py:22
│ │ │ │ ├─ 0.123 <module> pytz/__init__.py:9
│ │ │ │ │ └─ 0.110 <module> pkg_resources/__init__.py:15
│ │ │ │ │ ├─ 0.037 _call_aside pkg_resources/__init__.py:2938
│ │ │ │ │ │ └─ 0.037 _initialize_master_working_set pkg_resources/__init__.py:2953
│ │ │ │ │ └─ 0.036 load_module pkg_resources/extern/__init__.py:34
│ │ │ │ ├─ 0.114 <module> pandas/core/config_init.py:11
│ │ │ │ │ └─ 0.083 <module> pandas/formats/format.py:2
│ │ │ │ │ └─ 0.032 <module> pandas/core/index.py:2
│ │ │ │ └─ 0.067 <module> pandas/io/api.py:3
│ │ │ └─ 0.053 <module> tensorflow/contrib/learn/python/learn/estimators/linear.py:16
│ │ │ └─ 0.051 <module> tensorflow/contrib/linear_optimizer/__init__.py:20
│ │ │ └─ 0.043 <module> tensorflow/contrib/linear_optimizer/python/ops/sdca_ops.py:15
│ │ └─ 0.033 <module> tensorflow/contrib/learn/python/learn/dataframe/__init__.py:16
│ ├─ 0.711 <module> tensorflow/contrib/distributions/__init__.py:73
│ │ ├─ 0.508 <module> tensorflow/contrib/distributions/python/ops/chi2.py:15
│ │ │ └─ 0.506 <module> tensorflow/contrib/distributions/python/ops/gamma.py:15
│ │ │ └─ 0.506 <module> tensorflow/contrib/framework/__init__.py:58
│ │ │ └─ 0.498 <module> tensorflow/contrib/framework/python/ops/__init__.py:15
│ │ │ └─ 0.489 <module> tensorflow/contrib/framework/python/ops/embedding_ops.py:15
│ │ │ └─ 0.487 <module> tensorflow/contrib/layers/__init__.py:79
│ │ │ └─ 0.482 <module> tensorflow/contrib/layers/python/layers/__init__.py:15
│ │ │ ├─ 0.172 <module> tensorflow/contrib/layers/python/layers/layers.py:17
│ │ │ │ └─ 0.160 <module> tensorflow/python/ops/standard_ops.py:17
│ │ │ │ └─ 0.061 <module> tensorflow/python/ops/gradients.py:15
│ │ │ ├─ 0.131 <module> tensorflow/contrib/layers/python/layers/optimizers.py:15
│ │ │ │ └─ 0.127 <module> tensorflow/python/training/training.py:137
│ │ │ │ └─ 0.035 <module> tensorflow/python/training/adadelta.py:16
│ │ │ │ └─ 0.035 <module> tensorflow/python/training/training_ops.py:16
│ │ │ ├─ 0.069 <module> tensorflow/contrib/layers/python/layers/feature_column.py:68
│ │ │ ├─ 0.053 <module> tensorflow/contrib/layers/python/layers/embedding_ops.py:15
│ │ │ │ └─ 0.050 <module> tensorflow/contrib/layers/python/ops/sparse_feature_cross_op.py:15
│ │ │ │ └─ 0.045 load_op_library tensorflow/python/framework/load_library.py:40
│ │ │ └─ 0.048 <module> tensorflow/contrib/layers/python/layers/target_column.py:16
│ │ │ └─ 0.046 <module> tensorflow/contrib/metrics/__init__.py:135
│ │ │ └─ 0.039 <module> tensorflow/contrib/metrics/python/ops/metric_ops.py:19
│ │ │ └─ 0.037 <module> tensorflow/contrib/metrics/python/ops/set_ops.py:15
│ │ │ └─ 0.034 load_op_library tensorflow/python/framework/load_library.py:40
│ │ └─ 0.161 <module> tensorflow/contrib/distributions/python/ops/bernoulli.py:15
│ │ └─ 0.158 <module> tensorflow/python/ops/nn.py:271
│ │ └─ 0.087 <module> tensorflow/python/ops/init_ops.py:16
│ │ └─ 0.085 <module> tensorflow/python/ops/nn_ops.py:15
│ │ └─ 0.060 <module> tensorflow/python/ops/gen_nn_ops.py:4
│ │ └─ 0.057 _InitOpDefLibrary tensorflow/python/ops/gen_nn_ops.py:1630
│ │ └─ 0.054 Merge google/protobuf/text_format.py:291
│ │ └─ 0.052 MergeLines google/protobuf/text_format.py:331
│ │ └─ 0.052 _ParseOrMerge google/protobuf/text_format.py:350
│ │ └─ 0.052 _MergeField google/protobuf/text_format.py:374
│ │ └─ 0.052 _MergeField google/protobuf/text_format.py:374
│ │ └─ 0.038 _MergeField google/protobuf/text_format.py:374
│ ├─ 0.265 <module> tensorflow/contrib/bayesflow/__init__.py:18
│ │ └─ 0.264 <module> tensorflow/contrib/bayesflow/python/ops/stochastic_graph.py:38
│ │ ├─ 0.145 <module> tensorflow/python/ops/array_ops.py:70
│ │ │ ├─ 0.068 <module> tensorflow/python/ops/gen_math_ops.py:4
│ │ │ │ └─ 0.065 _InitOpDefLibrary tensorflow/python/ops/gen_math_ops.py:2378
│ │ │ │ └─ 0.063 Merge google/protobuf/text_format.py:291
│ │ │ │ └─ 0.063 MergeLines google/protobuf/text_format.py:331
│ │ │ │ └─ 0.063 _ParseOrMerge google/protobuf/text_format.py:350
│ │ │ │ └─ 0.063 _MergeField google/protobuf/text_format.py:374
│ │ │ │ └─ 0.059 _MergeField google/protobuf/text_format.py:374
│ │ │ │ └─ 0.052 _MergeField google/protobuf/text_format.py:374
│ │ │ └─ 0.045 <module> tensorflow/python/ops/gen_array_ops.py:4
│ │ │ └─ 0.039 _InitOpDefLibrary tensorflow/python/ops/gen_array_ops.py:2677
│ │ │ └─ 0.038 Merge google/protobuf/text_format.py:291
│ │ │ └─ 0.038 MergeLines google/protobuf/text_format.py:331
│ │ │ └─ 0.038 _ParseOrMerge google/protobuf/text_format.py:350
│ │ │ └─ 0.038 _MergeField google/protobuf/text_format.py:374
│ │ │ └─ 0.035 _MergeField google/protobuf/text_format.py:374
│ │ └─ 0.085 <module> tensorflow/python/ops/math_ops.py:210
│ ├─ 0.071 <module> tensorflow/contrib/slim/__init__.py:18
│ │ └─ 0.046 <module> tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py:20
│ │ └─ 0.036 TFExampleDecoder tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py:273
│ ├─ 0.051 <module> tensorflow/contrib/quantization/__init__.py:16
│ │ └─ 0.050 <module> tensorflow/contrib/quantization/python/__init__.py:15
│ └─ 0.045 <module> tensorflow/contrib/copy_graph/__init__.py:20
│ └─ 0.043 <module> tensorflow/contrib/copy_graph/python/util/copy_elements.py:27
├─ 0.299 <module> numpy/__init__.py:106
│ └─ 0.235 <module> numpy/add_newdocs.py:10
│ └─ 0.230 <module> numpy/lib/__init__.py:1
│ └─ 0.160 <module> numpy/lib/type_check.py:3
│ └─ 0.158 <module> numpy/core/__init__.py:1
│ └─ 0.036 <module> numpy/testing/__init__.py:7
├─ 0.151 <module> tensorflow/python/pywrap_tensorflow.py:11
│ └─ 0.148 swig_import_helper tensorflow/python/pywrap_tensorflow.py:13
├─ 0.072 <module> tensorflow/core/framework/graph_pb2.py:4
└─ 0.039 <module> tensorflow/python/platform/test.py:57
答案 0 :(得分：1)
我对TensorFlow了解不多，但我认为问题是process_image使用了一堆全局变量，特别是tf。每当它被称为你在一组不断增加的图像上运行TensorFlow。首先是1, then 2, then 3, then 4, 5, 6, ...
1 + 2 + 3 + 4 + 5 + ... + n = n ( n + 1 ) / 2
所以，通过100张图片，你实际上已经处理了5,050张。这是一个O（n 2 ）算法，这意味着它的运行时（在这种情况下，内存）将随着图像数量的增加呈指数增长。
同样，我对TensorFlow了解不多，但也许留下来调用sess.run更有意义？虽然您似乎对中间结果感兴趣？
而且，作为一个非常好的经验法则，避免使用全局变量。很难将它们与局部变量区分开来，它们打破了功能的整齐封装，使程序难以理解，并且导致了这样的累积问题。