Source code for Module.FN_Computation_torch

# Yuncong Ma, 6/5/2024
# FN Computation module of pNet
# Pytorch version

#########################################
# Packages
import scipy.io as sio

# other functions of pNet
from Module.Data_Input import *
from Module.FN_Computation import check_gFN, bootstrap_scan, setup_pFN_folder
import Module.GIG_ICA as GIG_ICA
import Module.SR_NMF as SR_NMF
from Basic.Cluster_Computation import submit_bash_job


[docs]class setup_FN_Computation: setup_SR_NMF = SR_NMF.setup_SR_NMF setup_GIG_ICA = GIG_ICA.setup_GIG_ICA
[docs]def run_FN_Computation_torch(dir_pnet_result: str): """ run the FN Computation module with settings ready in Data_Input and FN_Computation :param dir_pnet_result: directory of pNet result folder Yuncong Ma, 2/8/2024 """ # get directories of sub-folders dir_pnet_dataInput, dir_pnet_FNC, dir_pnet_gFN, dir_pnet_pFN, _, _ = setup_result_folder(dir_pnet_result) # log file logFile_FNC = os.path.join(dir_pnet_FNC, 'log.log') logFile_FNC = open(logFile_FNC, 'w') print('\nStart FN computation using PyTorch at ' + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) + '\n', file=logFile_FNC, flush=True) # load settings for data input and FN computation if not os.path.isfile(os.path.join(dir_pnet_dataInput, 'Setting.json')): raise ValueError('Cannot find the setting json file in folder Data_Input') if not os.path.isfile(os.path.join(dir_pnet_FNC, 'Setting.json')): raise ValueError('Cannot find the setting json file in folder FN_Computation') settingDataInput = load_json_setting(os.path.join(dir_pnet_dataInput, 'Setting.json')) settingFNC = load_json_setting(os.path.join(dir_pnet_FNC, 'Setting.json')) setting = {'Data_Input': settingDataInput, 'FN_Computation': settingFNC} print('Settings are loaded from folder Data_Input and FN_Computation', file=logFile_FNC, flush=True) # load basic settings dataType = setting['Data_Input']['Data_Type'] dataFormat = setting['Data_Input']['Data_Format'] # load Brain Template Brain_Template = load_brain_template(os.path.join(dir_pnet_dataInput, 'Brain_Template.json.zip')) if dataType == 'Volume': Brain_Mask = Brain_Template['Brain_Mask'] else: Brain_Mask = None print('Brain template is loaded from folder Data_Input', file=logFile_FNC, flush=True) # FN method FN_Method = setting['FN_Computation']['Method'] if FN_Method == 'SR-NMF': # ============== SR-NMF ============== # print('FN computation uses sparsity-regularized non-negative matrix factorization method', file=logFile_FNC, flush=True) # Generate additional parameters gNb = SR_NMF.compute_gNb(Brain_Template) scipy.io.savemat(os.path.join(dir_pnet_FNC, 'gNb.mat'), {'gNb': gNb}, do_compression=True) # ============== gFN Computation ============== # if setting['FN_Computation']['Group_FN']['file_gFN'] is None: # 2 steps # step 1 ============== bootstrap # sub-folder in FNC for storing bootstrapped results print('Start to prepare bootstrap files', file=logFile_FNC, flush=True) dir_pnet_BS = os.path.join(dir_pnet_FNC, 'BootStrapping') if not os.path.exists(dir_pnet_BS): os.makedirs(dir_pnet_BS) # Log logFile = os.path.join(dir_pnet_BS, 'Log_gFN_SR_NMF.log') # Input files file_scan = os.path.join(dir_pnet_dataInput, 'Scan_List.txt') file_subject_ID = os.path.join(dir_pnet_dataInput, 'Subject_ID.txt') file_subject_folder = os.path.join(dir_pnet_dataInput, 'Subject_Folder.txt') file_group_ID = os.path.join(dir_pnet_dataInput, 'Group_ID.txt') if not os.path.exists(file_group_ID): file_group = None # Parameters combineScan = setting['FN_Computation']['Combine_Scan'] init = setting['FN_Computation']['Group_FN']['BootStrap']['init'] #added on 08/03/2024 samplingMethod = setting['FN_Computation']['Group_FN']['BootStrap']['samplingMethod'] sampleSize = setting['FN_Computation']['Group_FN']['BootStrap']['sampleSize'] nBS = setting['FN_Computation']['Group_FN']['BootStrap']['nBS'] nTPoints = setting['FN_Computation']['Group_FN']['BootStrap']['nTPoints'] #added on 08/01/2024 # create scan lists for bootstrap bootstrap_scan(dir_pnet_BS, file_scan, file_subject_ID, file_subject_folder, file_group_ID=file_group_ID, combineScan=combineScan, samplingMethod=samplingMethod, sampleSize=sampleSize, nBS=nBS, logFile=logFile) # Parameters K = setting['FN_Computation']['K'] maxIter = setting['FN_Computation']['Group_FN']['maxIter'] minIter = setting['FN_Computation']['Group_FN']['minIter'] error = setting['FN_Computation']['Group_FN']['error'] normW = setting['FN_Computation']['Group_FN']['normW'] Alpha = setting['FN_Computation']['Group_FN']['Alpha'] Beta = setting['FN_Computation']['Group_FN']['Beta'] alphaS = setting['FN_Computation']['Group_FN']['alphaS'] alphaL = setting['FN_Computation']['Group_FN']['alphaL'] vxI = setting['FN_Computation']['Group_FN']['vxI'] ard = setting['FN_Computation']['Group_FN']['ard'] eta = setting['FN_Computation']['Group_FN']['eta'] nRepeat = setting['FN_Computation']['Group_FN']['nRepeat'] dataPrecision = setting['FN_Computation']['Computation']['dataPrecision'] # separate maxIter and minIter for gFN and pFN if isinstance(maxIter, int) or (isinstance(maxIter, np.ndarray) and maxIter.shape == 1): maxIter_gFN = maxIter maxIter_pFN = maxIter else: maxIter_gFN = maxIter[0] maxIter_pFN = maxIter[1] if isinstance(minIter, int) or (isinstance(minIter, np.ndarray) and minIter.shape == 1): minIter_gFN = minIter minIter_pFN = minIter else: minIter_gFN = minIter[0] minIter_pFN = minIter[1] # NMF on bootstrapped subsets print('Start to NMF for each bootstrap at ' + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())), file=logFile_FNC, flush=True) dir_pnet_BS = os.path.join(dir_pnet_FNC, 'BootStrapping') K = setting['FN_Computation']['K'] nBS = setting['FN_Computation']['Group_FN']['BootStrap']['nBS'] for rep in range(1, 1+nBS): # log file logFile = os.path.join(dir_pnet_BS, str(rep), 'Log_gFN_SR_NMF.log') # load data file_scan_list = os.path.join(dir_pnet_BS, str(rep), 'Scan_List.txt') Data, CHeader, NHeader = load_fmri_scan(file_scan_list, dataType=dataType, dataFormat=dataFormat, nTPoints=nTPoints, Reshape=True, Brain_Mask=Brain_Mask, Normalization='vp-vmax', logFile=logFile) # perform NMF FN_BS = SR_NMF.gFN_SR_NMF_torch(Data, K, gNb, init=init, maxIter=maxIter_gFN, minIter=minIter_gFN, error=error, normW=normW, Alpha=Alpha, Beta=Beta, alphaS=alphaS, alphaL=alphaL, vxI=vxI, ard=ard, eta=eta, nRepeat=nRepeat, dataPrecision=dataPrecision, logFile=logFile) # save results FN_BS = reshape_FN(FN_BS.numpy(), dataType=dataType, Brain_Mask=Brain_Mask) sio.savemat(os.path.join(dir_pnet_BS, str(rep), 'FN.mat'), {"FN": FN_BS}, do_compression=True) # save FNs in nii.gz and TC as txt file FY 07/26/2024 output_FN(FN=FN_BS, file_output=os.path.join(dir_pnet_BS, str(rep), 'FN.mat'), file_brain_template = Brain_Template, dataFormat=dataFormat, Cheader = CHeader, Nheader = NHeader) # step 2 ============== fuse results # Generate gFNs print('Start to fuse bootstrapped results using NCut at ' + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())), file=logFile_FNC, flush=True) dir_pnet_BS = os.path.join(dir_pnet_FNC, 'BootStrapping') K = setting['FN_Computation']['K'] nBS = setting['FN_Computation']['Group_FN']['BootStrap']['nBS'] FN_BS = np.empty(nBS, dtype=np.ndarray) # load bootstrapped results for rep in range(1, nBS+1): FN_BS[rep-1] = np.array(reshape_FN(load_matlab_single_array(os.path.join(dir_pnet_BS, str(rep), 'FN.mat')), dataType=dataType, Brain_Mask=Brain_Mask)) gFN_BS = np.concatenate(FN_BS, axis=1) # log logFile = os.path.join(dir_pnet_gFN, 'Log.log') # Fuse bootstrapped results gFN = SR_NMF.gFN_fusion_NCut_torch(gFN_BS, K, logFile=logFile) # output gFN = reshape_FN(gFN.numpy(), dataType=dataType, Brain_Mask=Brain_Mask) sio.savemat(os.path.join(dir_pnet_gFN, 'FN.mat'), {"FN": gFN}, do_compression=True) # save FNs in nii.gz and TC as txt file FY 07/26/2024 output_FN(FN=gFN, file_output=os.path.join(dir_pnet_gFN, 'FN.mat'), file_brain_template = Brain_Template, dataFormat=dataFormat, Cheader = CHeader, Nheader = NHeader) else: # use precomputed gFNs file_gFN = setting['FN_Computation']['Group_FN']['file_gFN'] gFN = load_matlab_single_array(file_gFN) if dataType == 'Volume': Brain_Mask = load_brain_template(os.path.join(dir_pnet_dataInput, 'Brain_Template.json.zip'))['Brain_Mask'] gFN = reshape_FN(gFN, dataType=dataType, Brain_Mask=Brain_Mask) check_gFN(gFN, method=FN_Method, logFile=logFile_FNC) if dataType == 'Volume': gFN = reshape_FN(gFN, dataType=dataType, Brain_Mask=Brain_Mask) sio.savemat(os.path.join(dir_pnet_gFN, 'FN.mat'), {"FN": gFN}, do_compression=True) # save FNs in nii.gz and TC as txt file FY 07/26/2024 # won't save imaging data for pre-computed Group FNs #output_FN(FN=gFN, # file_output=os.path.join(dir_pnet_gFN, 'FN.mat'), # file_brain_template = Brain_Template, # dataFormat=dataFormat) print('load precomputed gFNs', file=logFile_FNC, flush=True) # ============================================= # # ============== pFN Computation ============== # print('Start to compute pFNs at ' + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())), file=logFile_FNC, flush=True) # load precomputed gFNs gFN = load_matlab_single_array(os.path.join(dir_pnet_gFN, 'FN.mat')) # additional parameter gNb = load_matlab_single_array(os.path.join(dir_pnet_FNC, 'gNb.mat')) # reshape to 2D if required gFN = reshape_FN(gFN, dataType=dataType, Brain_Mask=Brain_Mask) # setup folders in Personalized_FN list_subject_folder = setup_pFN_folder(dir_pnet_result) N_Scan = len(list_subject_folder) for i in range(1, N_Scan+1): print(f'Start to compute pFNs for {i}-th folder: {list_subject_folder[i-1]}', file=logFile_FNC, flush=True) dir_pnet_pFN_indv = os.path.join(dir_pnet_pFN, list_subject_folder[i-1]) # parameter maxIter = setting['FN_Computation']['Personalized_FN']['maxIter'] minIter = setting['FN_Computation']['Personalized_FN']['minIter'] meanFitRatio = setting['FN_Computation']['Personalized_FN']['meanFitRatio'] error = setting['FN_Computation']['Personalized_FN']['error'] normW = setting['FN_Computation']['Personalized_FN']['normW'] Alpha = setting['FN_Computation']['Personalized_FN']['Alpha'] Beta = setting['FN_Computation']['Personalized_FN']['Beta'] alphaS = setting['FN_Computation']['Personalized_FN']['alphaS'] alphaL = setting['FN_Computation']['Personalized_FN']['alphaL'] vxI = setting['FN_Computation']['Personalized_FN']['vxI'] ard = setting['FN_Computation']['Personalized_FN']['ard'] eta = setting['FN_Computation']['Personalized_FN']['eta'] dataPrecision = setting['FN_Computation']['Computation']['dataPrecision'] # separate maxIter and minIter for gFN and pFN if isinstance(maxIter, int) or (isinstance(maxIter, np.ndarray) and maxIter.shape == 1): maxIter_gFN = maxIter maxIter_pFN = maxIter else: maxIter_gFN = maxIter[0] maxIter_pFN = maxIter[1] if isinstance(minIter, int) or (isinstance(minIter, np.ndarray) and minIter.shape == 1): minIter_gFN = minIter minIter_pFN = minIter else: minIter_gFN = minIter[0] minIter_pFN = minIter[1] # log file logFile = os.path.join(dir_pnet_pFN_indv, 'Log_pFN_SR_NMF.log') # load data Data, CHeader, NHeader = load_fmri_scan(os.path.join(dir_pnet_pFN_indv, 'Scan_List.txt'), dataType=dataType, dataFormat=dataFormat, Reshape=True, Brain_Mask=Brain_Mask, logFile=logFile) # perform NMF TC, pFN = SR_NMF.pFN_SR_NMF_torch(Data, gFN, gNb, maxIter=maxIter_pFN, minIter=minIter_pFN, meanFitRatio=meanFitRatio, error=error, normW=normW, Alpha=Alpha, Beta=Beta, alphaS=alphaS, alphaL=alphaL, vxI=vxI, ard=ard, eta=eta, dataPrecision=dataPrecision, logFile=logFile) pFN = pFN.numpy() TC = TC.numpy() # output pFN = reshape_FN(pFN, dataType=dataType, Brain_Mask=Brain_Mask) sio.savemat(os.path.join(dir_pnet_pFN_indv, 'FN.mat'), {"FN": pFN}, do_compression=True) sio.savemat(os.path.join(dir_pnet_pFN_indv, 'TC.mat'), {"TC": TC}, do_compression=True) # save FNs in nii.gz and TC as txt file FY 07/26/2024 output_FN(FN=pFN, file_output=os.path.join(dir_pnet_pFN_indv, 'FN.mat'), file_brain_template = Brain_Template, dataFormat=dataFormat, Cheader = CHeader, Nheader = NHeader) np.savetxt(os.path.join(dir_pnet_pFN_indv, 'TC.txt'), TC) elif FN_Method == 'GIG-ICA': # ============== GIG-ICA ============== # # gFNs must be loaded if setting['FN_Computation']['Group_FN']['file_gFN'] is None: print(f'\nError: \nCannot find the group-level FNs for GIG-ICA ', file=logFile_FNC, flush=True) raise ValueError('Require gFNs for GIG-ICA') file_gFN = setting['FN_Computation']['Group_FN']['file_gFN'] #gFN = load_matlab_single_array(file_gFN) # have to be 'Volume' gFN, _, _ = load_fmri_scan(file_gFN, dataType='Volume', dataFormat='Volume (*.nii, *.nii.gz, *.mat)', Reshape=False, Normalization=None) if dataType == 'Volume': Brain_Mask = load_brain_template(os.path.join(dir_pnet_dataInput, 'Brain_Template.json.zip'))['Brain_Mask'] gFN = reshape_FN(gFN, dataType=dataType, Brain_Mask=Brain_Mask) check_gFN(gFN, method=FN_Method, logFile=logFile_FNC) if dataType == 'Volume': gFN = reshape_FN(gFN, dataType=dataType, Brain_Mask=Brain_Mask) sio.savemat(os.path.join(dir_pnet_gFN, 'FN.mat'), {"FN": gFN}, do_compression=True) # save FNs in nii.gz and TC as txt file FY 07/26/2024 # won't save precomputed gFN #output_FN(FN=gFN, # file_output=os.path.join(dir_pnet_gFN, 'FN.mat'), # file_brain_template = Brain_Template, # dataFormat=dataFormat) print('load precomputed gFNs', file=logFile_FNC, flush=True) # ============== pFN Computation ============== # print('Start to compute pFNs at ' + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())), file=logFile_FNC, flush=True) # load precomputed gFNs gFN = load_matlab_single_array(os.path.join(dir_pnet_gFN, 'FN.mat')) # reshape to 2D if required gFN = reshape_FN(gFN, dataType=dataType, Brain_Mask=Brain_Mask) # setup folders in Personalized_FN list_subject_folder = setup_pFN_folder(dir_pnet_result) N_Scan = len(list_subject_folder) for i in range(1, N_Scan+1): print(f'Start to compute pFNs for {i}-th folder: {list_subject_folder[i-1]}', file=logFile_FNC, flush=True) dir_pnet_pFN_indv = os.path.join(dir_pnet_pFN, list_subject_folder[i-1]) # parameter maxIter = setting['FN_Computation']['Personalized_FN']['maxIter'] a = setting['FN_Computation']['Personalized_FN']['a'] Nemda = setting['FN_Computation']['Personalized_FN']['Nemda'] ftol = setting['FN_Computation']['Personalized_FN']['ftol'] error = setting['FN_Computation']['Personalized_FN']['error'] dataPrecision = setting['FN_Computation']['Computation']['dataPrecision'] # separate maxIter and minIter for gFN and pFN if isinstance(maxIter, int) or (isinstance(maxIter, np.ndarray) and maxIter.shape == 1): maxIter_gFN = maxIter maxIter_pFN = maxIter else: maxIter_gFN = maxIter[0] maxIter_pFN = maxIter[1] # log file logFile = os.path.join(dir_pnet_pFN_indv, 'Log_pFN_GIG_ICA.log') # load data Data, CHeader, NHeader = load_fmri_scan(os.path.join(dir_pnet_pFN_indv, 'Scan_List.txt'), dataType=dataType, dataFormat=dataFormat, Reshape=True, Brain_Mask=Brain_Mask, logFile=logFile) # perform GIG-ICA TC, pFN = GIG_ICA.pFN_GIG_ICA_torch(Data, gFN, maxIter=maxIter_pFN, a=a, Nemda=Nemda, ftol=ftol, error=error, dataPrecision=dataPrecision, logFile=logFile) if not isinstance(pFN, np.ndarray): pFN = pFN.numpy() TC = TC.numpy() # output pFN = reshape_FN(pFN, dataType=dataType, Brain_Mask=Brain_Mask) sio.savemat(os.path.join(dir_pnet_pFN_indv, 'FN.mat'), {"FN": pFN}, do_compression=True) sio.savemat(os.path.join(dir_pnet_pFN_indv, 'TC.mat'), {"TC": TC}, do_compression=True) # save FNs in nii.gz and TC as txt file FY 07/26/2024 output_FN(FN=pFN, file_output=os.path.join(dir_pnet_pFN_indv, 'FN.mat'), file_brain_template = Brain_Template, dataFormat=dataFormat, Cheader = CHeader, Nheader = NHeader) np.savetxt(os.path.join(dir_pnet_pFN_indv, 'TC.txt'), TC) # ============================================= # print('Finished FN computation at ' + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())), file=logFile_FNC, flush=True)
[docs]def run_FN_computation_torch_cluster(dir_pnet_result: str): """ run the FN Computation module in cluster :param dir_pnet_result: directory of pNet result folder Yuncong Ma, 6/5/2024 """ # get directories of sub-folders dir_pnet_dataInput, dir_pnet_FNC, dir_pnet_gFN, dir_pnet_pFN, _, _ = setup_result_folder(dir_pnet_result) # log file print('\nStart FN computation using PyTorch at ' + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) + '\n', flush=True) # load settings for data input and FN computation if not os.path.isfile(os.path.join(dir_pnet_dataInput, 'Setting.json')): raise ValueError('Cannot find the setting json file in folder Data_Input') if not os.path.isfile(os.path.join(dir_pnet_FNC, 'Setting.json')): raise ValueError('Cannot find the setting json file in folder FN_Computation') settingDataInput = load_json_setting(os.path.join(dir_pnet_dataInput, 'Setting.json')) settingCluster = load_json_setting(os.path.join(dir_pnet_dataInput, 'Cluster_Setting.json')) settingFNC = load_json_setting(os.path.join(dir_pnet_FNC, 'Setting.json')) setting = {'Data_Input': settingDataInput, 'FN_Computation': settingFNC, 'Cluster': settingCluster} print('Settings are loaded from folder Data_Input, FN_Computation and Cluster', flush=True) # load basic settings dataType = setting['Data_Input']['Data_Type'] dataFormat = setting['Data_Input']['Data_Format'] # load Brain Template Brain_Template = load_brain_template(os.path.join(dir_pnet_dataInput, 'Brain_Template.json.zip')) if dataType == 'Volume': Brain_Mask = Brain_Template['Brain_Mask'] else: Brain_Mask = None print('Brain template is loaded from folder Data_Input', flush=True) # FN method FN_Method = setting['FN_Computation']['Method'] if FN_Method == 'SR-NMF': # ============== SR-NMF ============== # print('FN computation uses spatial-regularized non-negative matrix factorization method', flush=True) # Generate additional parameters if not os.path.isfile(os.path.join(dir_pnet_FNC, 'gNb.mat')): gNb = SR_NMF.compute_gNb(Brain_Template) scipy.io.savemat(os.path.join(dir_pnet_FNC, 'gNb.mat'), {'gNb': gNb}, do_compression=True) # ============== gFN Computation ============== # if setting['FN_Computation']['Group_FN']['file_gFN'] is None: # 2 steps # step 1 ============== bootstrap # sub-folder in FNC for storing bootstrapped results print('Start to prepare bootstrap files', flush=True) dir_pnet_BS = os.path.join(dir_pnet_FNC, 'BootStrapping') if not os.path.exists(dir_pnet_BS): os.makedirs(dir_pnet_BS) # Input files file_scan = os.path.join(dir_pnet_dataInput, 'Scan_List.txt') file_subject_ID = os.path.join(dir_pnet_dataInput, 'Subject_ID.txt') file_subject_folder = os.path.join(dir_pnet_dataInput, 'Subject_Folder.txt') file_group_ID = os.path.join(dir_pnet_dataInput, 'Group_ID.txt') if not os.path.exists(file_group_ID): file_group_ID = None # Parameters combineScan = setting['FN_Computation']['Combine_Scan'] samplingMethod = setting['FN_Computation']['Group_FN']['BootStrap']['samplingMethod'] sampleSize = setting['FN_Computation']['Group_FN']['BootStrap']['sampleSize'] nBS = setting['FN_Computation']['Group_FN']['BootStrap']['nBS'] # create scan lists for bootstrap flag_complete = np.zeros(nBS) for i in range(1, 1+nBS): if os.path.exists(os.path.join(dir_pnet_BS, str(i))) and os.path.isfile(os.path.join(dir_pnet_BS, str(i), 'Scan_List.txt')): flag_complete[i-1] = 1 if np.sum(flag_complete) < nBS: bootstrap_scan(dir_pnet_BS, file_scan, file_subject_ID, file_subject_folder, file_group_ID=file_group_ID, combineScan=combineScan, samplingMethod=samplingMethod, sampleSize=sampleSize, nBS=nBS, logFile=None) # submit jobs memory = setting['Cluster']['computation_resource']['memory_bootstrap'] n_thread = setting['Cluster']['computation_resource']['thread_bootstrap'] for rep in range(1, 1+nBS): time.sleep(0.1) if os.path.isfile(os.path.join(dir_pnet_BS, str(i), 'FN.mat')): continue submit_bash_job(dir_pnet_result, python_command=f'pnet.SR_NMF.gFN_SR_NMF_boostrapping_cluster(dir_pnet_result,{rep})', memory=memory, n_thread=n_thread, bashFile=os.path.join(dir_pnet_BS, str(rep), 'cluster_job_bootstrap.sh'), pythonFile=os.path.join(dir_pnet_BS, str(rep), 'cluster_job_bootstrap.py'), logFile=os.path.join(dir_pnet_BS, str(rep), 'cluster_job_bootstrap.log') ) # check completion wait_time = 300 flag_complete = np.zeros(nBS) dir_pnet_BS = os.path.join(dir_pnet_FNC, 'BootStrapping') report_interval = 12 Count = 0 while np.sum(flag_complete) < nBS: time.sleep(wait_time) Count += 1 if Count % report_interval == 0: print(f'--> Found {np.sum(flag_complete)} finished jobs out of {nBS} at ' + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())), flush=True) for rep in range(1, 1+nBS): if flag_complete[rep-1] == 0 and os.path.isfile(os.path.join(dir_pnet_BS, str(rep), 'FN.mat')): flag_complete[rep-1] = 1 # step 2 ============== fuse results # Generate gFNs print('Start to fuse bootstrapped results using NCut at ' + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())), flush=True) memory = setting['Cluster']['computation_resource']['memory_fusion'] n_thread = setting['Cluster']['computation_resource']['thread_fusion'] if not os.path.isfile(os.path.join(dir_pnet_gFN, 'FN.mat')): submit_bash_job(dir_pnet_result, python_command='pnet.SR_NMF.fuse_FN_cluster(dir_pnet_result)', memory=memory, n_thread=n_thread, bashFile=os.path.join(dir_pnet_BS, 'cluster_job_fusion.sh'), pythonFile=os.path.join(dir_pnet_BS, 'cluster_job_fusion.py'), logFile=os.path.join(dir_pnet_BS, 'cluster_job_fusion.log') ) # check completion wait_time = 60 report_interval = 30 flag_complete = 0 Count = 0 while flag_complete == 0: time.sleep(wait_time) Count += 1 if Count % report_interval == 0: print(f'--> Found {np.sum(flag_complete)} finished jobs out of 1 at ' + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())), flush=True) if os.path.isfile(os.path.join(dir_pnet_gFN, 'FN.mat')): flag_complete = 1 break else: # use precomputed gFNs file_gFN = setting['FN_Computation']['Group_FN']['file_gFN'] gFN = load_matlab_single_array(file_gFN) if dataType == 'Volume': Brain_Mask = load_brain_template(os.path.join(dir_pnet_dataInput, 'Brain_Template.json.zip'))['Brain_Mask'] gFN = reshape_FN(gFN, dataType=dataType, Brain_Mask=Brain_Mask) check_gFN(gFN, method=FN_Method) if dataType == 'Volume': gFN = reshape_FN(gFN, dataType=dataType, Brain_Mask=Brain_Mask) sio.savemat(os.path.join(dir_pnet_gFN, 'FN.mat'), {"FN": gFN}, do_compression=True) # save FNs in nii.gz and TC as txt file FY 07/26/2024 # won't save precomputed gFN #output_FN(FN=gFN, # file_output=os.path.join(dir_pnet_gFN, 'FN.mat'), # file_brain_template = Brain_Template, # dataFormat=dataFormat) print('load precomputed gFNs', flush=True) # ============================================= # # ============== pFN Computation ============== # print('Start to compute pFNs at ' + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())), flush=True) # setup folders in Personalized_FN list_subject_folder = setup_pFN_folder(dir_pnet_result) nScan = len(list_subject_folder) # submit jobs memory = setting['Cluster']['computation_resource']['memory_pFN'] n_thread = setting['Cluster']['computation_resource']['thread_pFN'] for scan in range(1, 1+nScan): time.sleep(0.1) if os.path.isfile(os.path.join(dir_pnet_pFN, list_subject_folder[scan-1], 'FN.mat')): continue submit_bash_job(dir_pnet_result, python_command=f'pnet.SR_NMF.pFN_SR_NMF_cluster(dir_pnet_result,{scan})', memory=memory, n_thread=n_thread, bashFile=os.path.join(dir_pnet_pFN, list_subject_folder[scan-1], 'cluster_job_pFN.sh'), pythonFile=os.path.join(dir_pnet_pFN, list_subject_folder[scan-1], 'cluster_job_pFN.py'), logFile=os.path.join(dir_pnet_pFN, list_subject_folder[scan-1], 'cluster_job_pFN.log') ) # check completion wait_time = 30 report_interval = 120 flag_complete = np.zeros(nScan) Count = 0 while np.sum(flag_complete) < nScan: time.sleep(wait_time) Count += 1 if Count % report_interval == 0: print(f'--> Found {np.sum(flag_complete)} finished jobs out of {nScan} at ' + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())), flush=True) for scan in range(1, 1+nScan): if flag_complete[scan-1] == 0 and os.path.isfile(os.path.join(dir_pnet_pFN, list_subject_folder[scan-1], 'FN.mat')): flag_complete[scan-1] = 1 # ============================================= # elif FN_Method == 'GIG-ICA': # ============== GIG-ICA ============== # # gFNs must be loaded if setting['FN_Computation']['Group_FN']['file_gFN'] is None: print(f'\nError: \nCannot find the group-level FNs for GIG-ICA ', flush=True) raise ValueError('Require gFNs for GIG-ICA') file_gFN = setting['FN_Computation']['Group_FN']['file_gFN'] #gFN = load_matlab_single_array(file_gFN) # have to be 'Volume' gFN = load_fmri_scan(file_gFN, dataType='Volume', dataFormat='Volume (*.nii, *.nii.gz, *.mat)', Reshape=False, Normalization=None) if dataType == 'Volume': Brain_Mask = load_brain_template(os.path.join(dir_pnet_dataInput, 'Brain_Template.json.zip'))['Brain_Mask'] gFN = reshape_FN(gFN, dataType=dataType, Brain_Mask=Brain_Mask) check_gFN(gFN, method=FN_Method) if dataType == 'Volume': gFN = reshape_FN(gFN, dataType=dataType, Brain_Mask=Brain_Mask) sio.savemat(os.path.join(dir_pnet_gFN, 'FN.mat'), {"FN": gFN}, do_compression=True) # save FNs in nii.gz and TC as txt file FY 07/26/2024 # won't save pre-computed gFNs #output_FN(FN=gFN, # file_output=os.path.join(dir_pnet_gFN, 'FN.mat'), # file_brain_template = Brain_Template, # dataFormat=dataFormat, Cheader = CHeader, Nheader = NHeader) print('load precomputed gFNs', flush=True) # ============== pFN Computation ============== # print('Start to compute pFNs at ' + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())), flush=True) # setup folders in Personalized_FN list_subject_folder = setup_pFN_folder(dir_pnet_result) nScan = len(list_subject_folder) # submit jobs memory = setting['Cluster']['computation_resource']['memory_pFN'] n_thread = setting['Cluster']['computation_resource']['thread_pFN'] for scan in range(1, 1 + nScan): time.sleep(0.1) if os.path.isfile(os.path.join(dir_pnet_pFN, list_subject_folder[scan - 1], 'FN.mat')): continue submit_bash_job(dir_pnet_result, python_command=f'pnet.GIG_ICA.pFN_GIG_ICA_cluster(dir_pnet_result,{scan})', memory=memory, n_thread=n_thread, bashFile=os.path.join(dir_pnet_pFN, list_subject_folder[scan - 1], 'cluster_job_pFN.sh'), pythonFile=os.path.join(dir_pnet_pFN, list_subject_folder[scan - 1], 'cluster_job_pFN.py'), logFile=os.path.join(dir_pnet_pFN, list_subject_folder[scan - 1], 'cluster_job_pFN.log') ) # check completion wait_time = 30 report_interval = 120 flag_complete = np.zeros(nScan) Count = 0 while np.sum(flag_complete) < nScan: time.sleep(wait_time) Count += 1 if Count % report_interval == 0: print(f'--> Found {np.sum(flag_complete)} finished jobs out of {nScan} at ' + time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time())), flush=True) for scan in range(1, 1 + nScan): if flag_complete[scan - 1] == 0 and os.path.isfile( os.path.join(dir_pnet_pFN, list_subject_folder[scan - 1], 'FN.mat')): flag_complete[scan - 1] = 1 # ============================================= # print('Finished FN computation at ' + time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())), flush=True)