提问人:Prashant Pandey 提问时间:3/16/2023 最后编辑:SwierPrashant Pandey 更新时间:3/16/2023 访问量:156
在 Apple Silicon M2 中运行训练模型时出现 Tensorflow 错误
Tensorflow error running a training model in apple silicon M2
问:
我正在使用适用于macos的Deepfakelab。 该模型在 CPU 上运行良好,但是当我选择 GPU 时,这是它抛出的错误。 问题是什么,我该如何解决?
systemMemory: 16.00 GB
maxCacheSize: 5.33 GB
device config <core.leras.nn.nn.DeviceConfig object at 0x128bbfca0> devices <core.leras.device.Devices object at 0x128bbfb80> worst devices [0]:[METAL][0.0/0.0]
Press enter in 2 seconds to override model settings.models opt device /GPU:0
archi_type liae
Initializing models: 0%| | 0/5 [00:00<?, ?it/s]
Error: Graph execution error:
Detected at node 'encoder/down1/downs_0/conv1/weight/Initializer/random_uniform/RandomUniform' defined at (most recent call last):
File "/Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/threading.py", line 930, in _bootstrap
self._bootstrap_inner()
File "/Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/threading.py", line 973, in _bootstrap_inner
self.run()
File "/Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/threading.py", line 910, in run
self._target(*self._args, **self._kwargs)
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/mainscripts/Trainer.py", line 46, in trainerThread
model = models.import_model(model_class_name)(
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/models/ModelBase.py", line 193, in __init__
self.on_initialize()
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/models/Model_SAEHD/Model.py", line 337, in on_initialize
self.src_dst_saveable_weights = self.encoder.get_weights() + self.inter_AB.get_weights() + self.inter_B.get_weights() + self.decoder.get_weights()
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 77, in get_weights
self.build()
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 65, in build
self._build_sub(v[name],name)
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 35, in _build_sub
layer.build()
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 65, in build
self._build_sub(v[name],name)
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 20, in _build_sub
self._build_sub(sublayer, f"{name}_{i}")
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 35, in _build_sub
layer.build()
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 65, in build
self._build_sub(v[name],name)
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 33, in _build_sub
layer.build_weights()
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/layers/Conv2D.py", line 61, in build_weights
self.weight = tf.get_variable("weight", (self.kernel_size,self.kernel_size,self.in_ch,self.out_ch), dtype=self.dtype, initializer=kernel_initializer, trainable=self.trainable )
Node: 'encoder/down1/downs_0/conv1/weight/Initializer/random_uniform/RandomUniform'
Detected at node 'encoder/down1/downs_0/conv1/weight/Initializer/random_uniform/RandomUniform' defined at (most recent call last):
File "/Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/threading.py", line 930, in _bootstrap
self._bootstrap_inner()
File "/Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/threading.py", line 973, in _bootstrap_inner
self.run()
File "/Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/threading.py", line 910, in run
self._target(*self._args, **self._kwargs)
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/mainscripts/Trainer.py", line 46, in trainerThread
model = models.import_model(model_class_name)(
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/models/ModelBase.py", line 193, in __init__
self.on_initialize()
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/models/Model_SAEHD/Model.py", line 337, in on_initialize
self.src_dst_saveable_weights = self.encoder.get_weights() + self.inter_AB.get_weights() + self.inter_B.get_weights() + self.decoder.get_weights()
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 77, in get_weights
self.build()
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 65, in build
self._build_sub(v[name],name)
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 35, in _build_sub
layer.build()
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 65, in build
self._build_sub(v[name],name)
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 20, in _build_sub
self._build_sub(sublayer, f"{name}_{i}")
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 35, in _build_sub
layer.build()
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 65, in build
self._build_sub(v[name],name)
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 33, in _build_sub
layer.build_weights()
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/layers/Conv2D.py", line 61, in build_weights
self.weight = tf.get_variable("weight", (self.kernel_size,self.kernel_size,self.in_ch,self.out_ch), dtype=self.dtype, initializer=kernel_initializer, trainable=self.trainable )
Node: 'encoder/down1/downs_0/conv1/weight/Initializer/random_uniform/RandomUniform'
Cannot assign a device for operation encoder/down1/downs_0/conv1/weight/Initializer/random_uniform/RandomUniform: Could not satisfy explicit device specification '' because the node {{colocation_node encoder/down1/downs_0/conv1/weight/Initializer/random_uniform/RandomUniform}} was colocated with a group of nodes that required incompatible device '/device:GPU:0'. All available devices [/job:localhost/replica:0/task:0/device:CPU:0, /job:localhost/replica:0/task:0/device:GPU:0].
Colocation Debug Info:
Colocation group had the following types and supported devices:
Root Member(assigned_device_name_index_=-1 requested_device_name_='/device:GPU:0' assigned_device_name_='' resource_device_name_='/device:GPU:0' supported_device_types_=[CPU] possible_devices_=[]
Identity: GPU CPU
Mul: GPU CPU
AddV2: GPU CPU
Sub: GPU CPU
RandomUniform: GPU CPU
Assign: CPU
VariableV2: GPU CPU
Const: GPU CPU
Colocation members, user-requested devices, and framework assigned devices, if any:
encoder/down1/downs_0/conv1/weight/Initializer/random_uniform/shape (Const)
encoder/down1/downs_0/conv1/weight/Initializer/random_uniform/min (Const)
encoder/down1/downs_0/conv1/weight/Initializer/random_uniform/max (Const)
encoder/down1/downs_0/conv1/weight/Initializer/random_uniform/RandomUniform (RandomUniform)
encoder/down1/downs_0/conv1/weight/Initializer/random_uniform/sub (Sub)
encoder/down1/downs_0/conv1/weight/Initializer/random_uniform/mul (Mul)
encoder/down1/downs_0/conv1/weight/Initializer/random_uniform (AddV2)
encoder/down1/downs_0/conv1/weight (VariableV2) /device:GPU:0
encoder/down1/downs_0/conv1/weight/Assign (Assign) /device:GPU:0
encoder/down1/downs_0/conv1/weight/read (Identity) /device:GPU:0
Assign_2 (Assign) /device:GPU:0
Assign_168 (Assign) /device:GPU:0
[[{{node encoder/down1/downs_0/conv1/weight/Initializer/random_uniform/RandomUniform}}]]
Original stack trace for 'encoder/down1/downs_0/conv1/weight/Initializer/random_uniform/RandomUniform':
File "/Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/threading.py", line 930, in _bootstrap
self._bootstrap_inner()
File "/Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/threading.py", line 973, in _bootstrap_inner
self.run()
File
Detected at node 'encoder/down1/downs_0/conv1/weight/Initializer/random_uniform/RandomUniform' defined at (most recent call last):
File "/Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/threading.py", line 930, in _bootstrap
self._bootstrap_inner()
File "/Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/threading.py", line 973, in _bootstrap_inner
self.run()
File "/Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/threading.py", line 910, in run
self._target(*self._args, **self._kwargs)
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/mainscripts/Trainer.py", line 46, in trainerThread
model = models.import_model(model_class_name)(
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/models/ModelBase.py", line 193, in __init__
self.on_initialize()
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/models/Model_SAEHD/Model.py", line 337, in on_initialize
self.src_dst_saveable_weights = self.encoder.get_weights() + self.inter_AB.get_weights() + self.inter_B.get_weights() + self.decoder.get_weights()
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 77, in get_weights
self.build()
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 65, in build
self._build_sub(v[name],name)
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 35, in _build_sub
layer.build()
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 65, in build
self._build_sub(v[name],name)
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 20, in _build_sub
self._build_sub(sublayer, f"{name}_{i}")
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 35, in _build_sub
layer.build()
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 65, in build
self._build_sub(v[name],name)
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 33, in _build_sub
layer.build_weights()
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/layers/Conv2D.py", line 61, in build_weights
self.weight = tf.get_variable("weight", (self.kernel_size,self.kernel_size,self.in_ch,self.out_ch), dtype=self.dtype, initializer=kernel_initializer, trainable=self.trainable )
Node: 'encoder/down1/downs_0/conv1/weight/Initializer/random_uniform/RandomUniform'
Detected at node 'encoder/down1/downs_0/conv1/weight/Initializer/random_uniform/RandomUniform' defined at (most recent call last):
File "/Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/threading.py", line 930, in _bootstrap
self._bootstrap_inner()
File "/Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/threading.py", line 973, in _bootstrap_inner
self.run()
File "/Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/threading.py", line 910, in run
self._target(*self._args, **self._kwargs)
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/mainscripts/Trainer.py", line 46, in trainerThread
model = models.import_model(model_class_name)(
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/models/ModelBase.py", line 193, in __init__
self.on_initialize()
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/models/Model_SAEHD/Model.py", line 337, in on_initialize
self.src_dst_saveable_weights = self.encoder.get_weights() + self.inter_AB.get_weights() + self.inter_B.get_weights() + self.decoder.get_weights()
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 77, in get_weights
self.build()
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 65, in build
self._build_sub(v[name],name)
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 35, in _build_sub
layer.build()
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 65, in build
self._build_sub(v[name],name)
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 20, in _build_sub
self._build_sub(sublayer, f"{name}_{i}")
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 35, in _build_sub
layer.build()
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 65, in build
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/layers/Conv2D.py", line 61, in build_weights
self.weight = tf.get_variable("weight", (self.kernel_size,self.kernel_size,self.in_ch,self.out_ch), dtype=self.dtype, initializer=kernel_initializer, trainable=self.trainable )
Node: 'encoder/down1/downs_0/conv1/weight/Initializer/random_uniform/RandomUniform'
Cannot assign a device for operation encoder/down1/downs_0/conv1/weight/Initializer/random_uniform/RandomUniform: Could not satisfy explicit device specification '' because the node {{colocation_node encoder/down1/downs_0/conv1/weight/Initializer/random_uniform/RandomUniform}} was colocated with a group of nodes that required incompatible device '/device:GPU:0'. All available devices [/job:localhost/replica:0/task:0/device:CPU:0, /job:localhost/replica:0/task:0/device:GPU:0].
Colocation Debug Info:
Colocation group had the following types and supported devices:
Root Member(assigned_device_name_index_=-1 requested_device_name_='/device:GPU:0' assigned_device_name_='' resource_device_name_='/device:GPU:0' supported_device_types_=[CPU] possible_devices_=[]
Identity: GPU CPU
Mul: GPU CPU
AddV2: GPU CPU
Sub: GPU CPU
RandomUniform: GPU CPU
Assign: CPU
VariableV2: GPU CPU
Const: GPU CPU
Colocation members, user-requested devices, and framework assigned devices, if any:
encoder/down1/downs_0/conv1/weight/Initializer/random_uniform/shape (Const)
encoder/down1/downs_0/conv1/weight/Initializer/random_uniform/min (Const)
encoder/down1/downs_0/conv1/weight/Initializer/random_uniform/max (Const)
encoder/down1/downs_0/conv1/weight/Initializer/random_uniform/RandomUniform (RandomUniform)
encoder/down1/downs_0/conv1/weight/Initializer/random_uniform/sub (Sub)
encoder/down1/downs_0/conv1/weight/Initializer/random_uniform/mul (Mul)
encoder/down1/downs_0/conv1/weight/Initializer/random_uniform (AddV2)
encoder/down1/downs_0/conv1/weight (VariableV2) /device:GPU:0
encoder/down1/downs_0/conv1/weight/Assign (Assign) /device:GPU:0
encoder/down1/downs_0/conv1/weight/read (Identity) /device:GPU:0
Assign_2 (Assign) /device:GPU:0
Assign_168 (Assign) /device:GPU:0
[[{{node encoder/down1/downs_0/conv1/weight/Initializer/random_uniform/RandomUniform}}]]
Original stack trace for 'encoder/down1/downs_0/conv1/weight/Initializer/random_uniform/RandomUniform':
File "/Applications/Xcode.app/Contents/Developer/Library/Frameworks/Python3.framework/Versions/3.9/lib/python3.9/threading.py", line 930, in _bootstrap
self._bootstrap_inner()
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 65, in build
self._build_sub(v[name],name)
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/models/ModelBase.py", line 33, in _build_sub
layer.build_weights()
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/.dfl/DeepFaceLab/core/leras/layers/Conv2D.py", line 61, in build_weights
self.weight = tf.get_variable("weight", (self.kernel_size,self.kernel_size,self.in_ch,self.out_ch), dtype=self.dtype, initializer=kernel_initializer, trainable=self.trainable )
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/lib/python3.9/site-packages/tensorflow/python/ops/variable_scope.py", line 1617, in get_variable
return get_variable_scter
return self._get_single_variable(
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/lib/python3.9/site-packages/tensorflow/python/ops/variable_scope.py", line 963, in _get_single_variable
v = variables.VariableV1(
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/lib/python3.9/site-packages/tensorflow/python/util/traceback_utils.py", line 150, in error_handler
return fn(*args, **kwargs)
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/lib/python3.9/site-packages/tensorflow/python/ops/variables.py", line 265, in __call__
return cls._variable_v1_call(*args, **kwargs)
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/lib/python3.9/site-packages/tensorflow/python/ops/variables.py", line 210, in _variable_v1_call
return previous_getter(
b/python3.9/site-packages/tensorflow/python/framework/ops.py", line 3776, in _create_op_internal
ret = Operation(
File "/Users/prashantpandey/Desktop/programming/deep_learning/dfl/DeepFaceLab_MacOS/lib/python3.9/site-packages/tensorflow/python/framework/ops.py", line 2175, in __init__
self._traceback = tf_stack.extract_stack_for_node(self._c_op)
我尝试在谷歌上搜索错误,但似乎没有人有解决方案,因此我选择在这里发帖。
答: 暂无答案
评论