传递 PySwig 对象的多处理

Multiprocessing passing a PySwig Object

提问人:Hans Baumann 提问时间:8/21/2023 更新时间:8/21/2023 访问量:34

问:

我正在尝试使用 PySAGA 模块在 python 中进行一些多处理。为了避免必须为每个工作线程加载输入栅格,我正在尝试将加载的栅格酸洗到多重处理任务中。到目前为止,我的代码看起来像这样:

import os,sys
import time, math
from joblib import Parallel, delayed
import time, math
import numpy as np
os.environ['SAGA_PATH'] = 'C:/Program Files/SAGA/'
os.add_dll_directory(os.environ['SAGA_PATH'])
sys.path.append('C:/Program Files/SAGA/')
def Run_Gravitational_Process_Path_Model(dem,source,mu,md):
    # Get the tool:
    Tool = saga.SG_Get_Tool_Library_Manager().Create_Tool('sim_geomorphology', '0')
    Verbose = False
    if not Verbose:
        saga.SG_UI_ProgressAndMsg_Lock(True)
    if not Tool:
        print('Failed to request tool: Gravitational Process Path Model')
        return False
    # Set the parameter interface:
    Tool.Reset()
    #p = Tool.Get_Parameters()
    Tool.Set_Parameter('DEM', dem)
    #p(saga.CSG_String('DEM')).Set_Value(dem)
    Tool.Set_Parameter('RELEASE_AREAS',source)
    #p(saga.CSG_String('RELEASE_AREA')).Set_Value(source)
    Tool.Set_Parameter('MATERIAL', saga.SG_Get_Data_Manager().Add('Grid input file, optional'))
    Tool.Set_Parameter('FRICTION_ANGLE_GRID', saga.SG_Get_Data_Manager().Add('Grid input file, optional'))
    Tool.Set_Parameter('SLOPE_IMPACT_GRID', saga.SG_Get_Data_Manager().Add('Grid input file, optional'))
    Tool.Set_Parameter('FRICTION_MU_GRID', saga.SG_Get_Data_Manager().Add('Grid input file, optional'))
    Tool.Set_Parameter('FRICTION_MASS_TO_DRAG_GRID', saga.SG_Get_Data_Manager().Add('Grid input file, optional'))
    Tool.Set_Parameter('OBJECTS', saga.SG_Get_Data_Manager().Add('Grid input file, optional'))
    Tool.Set_Parameter('DEPOSITION', saga.SG_Get_Create_Pointer()) # optional output, remove this line, if you don't want to create it
    Tool.Set_Parameter('MAX_VELOCITY', saga.SG_Get_Create_Pointer()) # optional output, remove this line, if you don't want to create it
    Tool.Set_Parameter('STOP_POSITIONS', saga.SG_Get_Create_Pointer()) # optional output, remove this line, if you don't want to create it
    Tool.Set_Parameter('ENDANGERED', saga.SG_Get_Create_Pointer()) # optional output, remove this line, if you don't want to create it
    Tool.Set_Parameter('PROCESS_PATH_MODEL', 1) # 'Random Walk'
    Tool.Set_Parameter('RW_SLOPE_THRES', 40)
    Tool.Set_Parameter('RW_EXPONENT', 2)
    Tool.Set_Parameter('RW_PERSISTENCE', 1.5)
    Tool.Set_Parameter('GPP_ITERATIONS', 1000)
    Tool.Set_Parameter('GPP_PROCESSING_ORDER', 2) # 'RAs in Parallel per Iteration'
    Tool.Set_Parameter('GPP_SEED', 1)
    Tool.Set_Parameter('FRICTION_MODEL', 5) # 'None'
    Tool.Set_Parameter('FRICTION_THRES_FREE_FALL', 60)
    Tool.Set_Parameter('FRICTION_METHOD_IMPACT', 0) # 'Energy Reduction (Scheidegger 1975)'
    Tool.Set_Parameter('FRICTION_IMPACT_REDUCTION', 75)
    Tool.Set_Parameter('FRICTION_ANGLE', 30)
    Tool.Set_Parameter('FRICTION_MU', mu)
    Tool.Set_Parameter('FRICTION_MODE_OF_MOTION', 0) # 'Sliding'
    Tool.Set_Parameter('FRICTION_MASS_TO_DRAG',md)
    Tool.Set_Parameter('FRICTION_INIT_VELOCITY', 1)
    Tool.Set_Parameter('DEPOSITION_MODEL', 0) # 'None'
    Tool.Set_Parameter('DEPOSITION_INITIAL', 20)
    Tool.Set_Parameter('DEPOSITION_SLOPE_THRES', 20)
    Tool.Set_Parameter('DEPOSITION_VELOCITY_THRES', 15)
    Tool.Set_Parameter('DEPOSITION_MAX', 20)
    Tool.Set_Parameter('DEPOSITION_MIN_PATH', 100)
    Tool.Set_Parameter('SINK_MIN_SLOPE', 2.5)
    # Execute the tool:
    if not Tool.Execute():
        print('failed to execute tool: ' + Tool.Get_Name().c_str())
        return False

    # Request the results:
    return Tool.Get_Parameter('PROCESS_AREA').asGrid(),\
           Tool.Get_Parameter('DEPOSITION').asGrid(), \
           Tool.Get_Parameter('MAX_VELOCITY').asGrid(), \
           Tool.Get_Parameter('STOP_POSITIONS').asGrid(), \
           Tool.Get_Parameter('ENDANGERED').asGrid(),

def load_data(path,dem_name,source_name):
    saga.SG_Set_History_Depth(0)  # History will not be created
    dem = 0
    source=0
    dem=saga.SG_Get_Data_Manager().Add_Grid('{}\{}.tif'.format(raster_input_path,dem_name))
    source=saga.SG_Get_Data_Manager().Add_Grid('{}\{}.tif'.format(raster_input_path,source_name))
    return dem,source



def run_GPP(job_number):
        mu = float(parameters[job_number][0])
        md = float(parameters[job_number][1])
        print('mu=', round(mu, 2), 'md=', md)
        result=Run_Gravitational_Process_Path_Model(dem,source,mu,md)
        result[0] .Save('{}_{}_{}.tif'.format(r"Downloads\Test\process_area",round(mu,2),round(md,2)))

mu_range=np.arange(0.2, 1, 0.1)
md_range=np.arange(200, 230, 2.5)
parameters=[]
for mu in mu_range:
    for md in md_range:
        parameters.append((mu,md))
number_of_jobs =len(parameters)
dem,source=load_data(raster_input_path,'dtm','release')

# Run in parallel with Joblib
start = time.time()
Parallel(n_jobs=-1,verbose=1)(delayed(run_GPP)(i) for i in range(number_of_jobs))
# when job is done, free memory resources:
saga.SG_Get_Data_Manager().Delete_All()
end = time.time()
print('{:.4f} s'.format(end-start))

但是,多处理任务失败,因为酸洗对象是 SWIG 对象。有人知道如何腌制 SWIG 对象吗?

多处理 swig joblib

评论

0赞 Flexo 9/3/2023
在这里,你可能会用共享内存做一些非常聪明的事情,使多重处理只是传递对共享内存的引用,而不是整个对象本身。
0赞 Hans Baumann 9/4/2023
我该怎么做?

答: 暂无答案