From f5c876163a10ecb9c38df63acdef011968616af4 Mon Sep 17 00:00:00 2001 From: piotrj Date: Tue, 23 Jan 2024 19:34:58 +0100 Subject: [PATCH] Where Is It? import operational --- README.md | 12 +- info/wii_import.md | 4 +- src/core.py | 511 ++++++++++++++++++++++++++++++++------------- src/librer.py | 86 ++++++-- 4 files changed, 433 insertions(+), 180 deletions(-) diff --git a/README.md b/README.md index f6b2820..bebcf55 100644 --- a/README.md +++ b/README.md @@ -16,16 +16,12 @@ The primary purpose of this software is to enable users to catalog their files, #### Context menu: ![image info](./info/menu.png) -## Download: -Portable executable packages created with [PyInstaller](https://pyinstaller.org/en/stable) for **Linux** and **Windows** can be downloaded from the Releases site: +## [Download](https://github.com/PJDude/librer/releases) ## +Portable executable packages created with [PyInstaller](https://pyinstaller.org/en/stable) for **Linux** and **Windows** can be downloaded from the Releases site. -https://github.com/PJDude/librer/releases +## [MAJORGEEKS review](https://www.majorgeeks.com/files/details/librer.html) ## -## MAJORGEEKS review: -https://www.majorgeeks.com/files/details/librer.html - -## SOFTPEDIA review: -https://www.softpedia.com/get/Others/File-CD-DVD-Catalog/Librer.shtml +## [SOFTPEDIA review](https://www.softpedia.com/get/Others/File-CD-DVD-Catalog/Librer.shtml) ## ## [Tutorial](./info/tutorial.md) ## diff --git a/info/wii_import.md b/info/wii_import.md index eed85fb..1068ab3 100644 --- a/info/wii_import.md +++ b/info/wii_import.md @@ -1,7 +1,5 @@ Librer allows to import data from "Where Is It?" software indirectly by reading it's xml report file. -### This feature is under development - How to import data ? You need to save information about all files catalogued by "Where Is It?" in "Where Is It?": @@ -17,6 +15,6 @@ in "Where Is It?": in Librer use File menu action: 'Import "Where Is It?" xml ...' ## unregistered Where Is It? version issue -Unregistered version of WII seems to pollute its own report by replacing some data with ```*** DEMO ***``` string. Imported data from such report will be obviously incomplete, however the ```*** DEMO ***``` string seems to be inserted randomly, so in sequential writes it appears in different sections. After couple of exports, with a little luck pool of report files will contain all data scattered in different files. Import all that reports at once in single shot (!) to merge the data into complete dataset. To do that use multiselection on import dialog. Incomplete data will be ignored and will not apear in librer records. +Unregistered version of WII pollutes its own report by replacing some data with ```*** DEMO ***``` string. Imported data from such report will be obviously incomplete, however the ```*** DEMO ***``` string seems to be inserted randomly, so in sequential writes it appears in different sections. After couple of exports, with a little luck pool of report files will contain all data scattered in different files. Import all that reports at once in single shot (!) to merge the data into complete dataset. To do that use multiselection on import dialog. Incomplete data will be ignored and will not apear in librer records. diff --git a/src/core.py b/src/core.py index b3a9899..9856f61 100644 --- a/src/core.py +++ b/src/core.py @@ -26,10 +26,12 @@ # #################################################################################### +from gc import disable as gc_disable, enable as gc_enable,collect as gc_collect + from json import loads as json_loads from subprocess import Popen, STDOUT,DEVNULL,PIPE, run as subprocess_run -from time import sleep, perf_counter,time,strftime,localtime +from time import sleep, perf_counter,time,strftime,localtime,mktime from threading import Thread from os import cpu_count,scandir,stat,sep,name as os_name,remove as os_remove,kill,rename @@ -46,6 +48,7 @@ from zipfile import ZipFile from platform import system as platform_system,release as platform_release,node as platform_node +from ciso8601 import parse_datetime from fnmatch import fnmatch from re import search as re_search import sys @@ -768,6 +771,36 @@ def threaded_cde(timeout_semi_list): self.header.cde_stats_time=customdata_stats_time self.header.cde_stats_time_all=customdata_stats_time_all[0] + ############################################################# + def sld_recalc_rec(self,scan_like_data): + new_size_on_this_level = 0 + new_files_quant_on_this_level = 0 + new_folders_quant_on_this_level = 0 + + for entry_name,items_list in scan_like_data.items(): + (size,is_dir,is_file,is_symlink,is_bind,has_files,mtime) = items_list[0:7] + + elem_index = 7 + if has_files: + sub_dict = items_list[elem_index] + elem_index+=1 + + sub_size,sub_quant,sub_folders_quant = self.sld_recalc_rec(sub_dict) + new_size_on_this_level+=sub_size + new_files_quant_on_this_level+=sub_quant + new_folders_quant_on_this_level+=sub_folders_quant+1 + + if size==0: + items_list[0]=sub_size + elif is_file: + new_files_quant_on_this_level+=1 + new_size_on_this_level+=size + else: + new_folders_quant_on_this_level+=1 + + #scan_like_data[0]=new_size_on_this_level + return new_size_on_this_level,new_files_quant_on_this_level,new_folders_quant_on_this_level + ############################################################# def tupelize_rec(self,scan_like_data,results_queue_put): LUT_encode_loc = LUT_encode @@ -1303,136 +1336,260 @@ def read_records_pre(self): return (0,0) def get_wii_files_dict(self,import_filenames): - re_obj_item = re_compile(r'') - re_obj_item_end = re_compile(r'/ITEM>') - - re_obj_name = re_compile(r'(.+)') - re_obj_ext = re_compile(r'(.+)') - re_obj_size = re_compile(r'(.+)') - re_obj_date = re_compile(r'(.+)') - re_obj_disk_name = re_compile(r'(.+)') - re_obj_disk_type = re_compile(r'(.+)') - re_obj_disk_num = re_compile(r'(.+)') - re_obj_disk_location = re_compile(r'(.+)') - re_obj_path = re_compile(r'(.+)') - re_obj_time = re_compile(r'') - re_obj_crc = re_compile(r'(.+)') - re_obj_category = re_compile(r'(.+)') - re_obj_flag = re_compile(r'(.+)') - re_obj_desc = re_compile(r'(.+)') - re_obj_desc_begin = re_compile(r'(.*)') - re_obj_desc_end = re_compile(r'(.*)') - - l=0 - in_item=False - in_description=False + # + re_obj_header_search = re_compile(r'.*<\?(.*)\?>.*').search - demo_str = '*** DEMO ***' + # + re_obj_comment_search = re_compile(r'.*.*').search - wii_paths_dict = {} - wii_path_tuple_to_data = {} + re_obj_report_search = re_compile(r'.*').search - try: - filenames_set=set() - for import_filename in import_filenames: - with open(import_filename,"rt", encoding='utf-8', errors='ignore') as f: - for line in f: - if match := re_obj_item.search(line): - item={} - item['type']=match.group(1) - item['ext'] = '' - item['description'] = [] + re_obj_report_end_search = re_compile(r'.*').search - in_item=True - in_description=False + re_obj_item_search = re_compile(r'.*').search + re_obj_item_end_search = re_compile(r'.*/ITEM>.*').search + re_obj_desc_search = re_compile(r'.*(.+).*').search + re_obj_desc_begin_search = re_compile(r'.*(.*)').search + re_obj_desc_end_search = re_compile(r'(.*).*').search - elif match := re_obj_item_end.search(line): - in_item=False - in_description=False + re_obj_name_search = re_compile(r'.*(.+).*').search + re_obj_ext_search = re_compile(r'.*(.+).*').search + re_obj_size_search = re_compile(r'.*(.+).*').search + re_obj_date_search = re_compile(r'.*(.+).*').search + re_obj_disk_name_search = re_compile(r'.*(.+).*').search + re_obj_disk_type_search = re_compile(r'.*(.+).*').search + re_obj_disk_num_search = re_compile(r'.*(.+).*').search + re_obj_disk_location_search = re_compile(r'.*(.+).*').search + re_obj_path_search = re_compile(r'.*(.+).*').search + re_obj_time_search = re_compile(r'.*.*').search + re_obj_crc_search = re_compile(r'.*(.+).*').search + re_obj_category_search = re_compile(r'.*(.+).*').search + re_obj_flag_search = re_compile(r'.*(.+).*').search - if item['disk_name']!=demo_str and item['path']!=demo_str and item['name']!=demo_str and item['ext']!=demo_str: - #path_splitted = [item['disk_name'] + ':'] + item['path'].strip('\\').split('\\') + [item['name']] - path_splitted = [item['disk_name']] + item['path'].strip('\\').split('\\') + [item['name'] + item['ext']] - path_splitted = [path_elem for path_elem in path_splitted if path_elem] + ####################################################################### - path_splitted_len = len(path_splitted) - path_splitted_tuple = tuple(path_splitted) + #l=0 + in_report=False + in_item=False + in_description=False - #print(f'{path_splitted=}') + demo_str = '*** DEMO ***' - next_dict = wii_paths_dict - for ps_i in range(path_splitted_len): - filename = path_splitted[ps_i] - filenames_set.add(filename) + wii_paths_dict = {} + wii_paths_dict_per_disk = defaultdict(dict) - if filename not in next_dict: - next_dict[filename] = {} + wii_path_tuple_to_data = {} + wii_path_tuple_to_data_per_disk = defaultdict(dict) - next_dict = next_dict[filename] + filenames_set=set() + filenames_set_per_disk=defaultdict(set) - size = item['size'] - is_dir = bool(item['type']=="Folder") - is_file = bool(item['type']=="File") - is_symlink=False - is_bind=False - has_files=False - mtime=1234 + filenames_set_add = filenames_set.add - #scan_like_data tuple - sld_tuple = tuple([size,is_dir,is_file,is_symlink,is_bind,has_files,mtime]) - #print(f'{path_splitted=} : {sld_tuple=}') + cd_set=set() + cd_set_per_disk=defaultdict(set) - wii_path_tuple_to_data[path_splitted_tuple] = sld_tuple + cd_set_add = cd_set.add - elif match := re_obj_name.search(line): - item['name']=match.group(1) - elif match := re_obj_ext.search(line): - ext = match.group(1) - if ext!=demo_str: - item['ext']='.' + ext - elif match := re_obj_size.search(line): - try: - item['size']=int(match.group(1)) - except: - #print(f'wrong size: {match.group(1)}') - item['size']=333222 - elif match := re_obj_date.search(line): - item['date']=match.group(1) - elif match := re_obj_disk_name.search(line): - item['disk_name']=match.group(1) - elif match := re_obj_disk_type.search(line): - item['disk_type']=match.group(1) - elif match := re_obj_disk_num.search(line): - item['disk_num']=match.group(1) - elif match := re_obj_disk_location.search(line): - item['disk_loc']=match.group(1) - elif match := re_obj_path.search(line): - item['path']=match.group(1) - elif match := re_obj_time.search(line): - item['time']=match.group(1) - elif match := re_obj_crc.search(line): - item['crc']=match.group(1) - elif match := re_obj_category.search(line): - item['category']=match.group(1) - elif match := re_obj_flag.search(line): - item['flag']=match.group(1) - elif match := re_obj_desc.search(line): - item['desc']=match.group(1) - elif match := re_obj_desc_begin.search(line): - in_description=True - item['description'].append(match.group(1)) - elif match := re_obj_desc_end.search(line): - in_description=False - item['description'].append(match.group(1)) - item['description'] = tuple(item['description']) - elif in_description: - item['description'].append(line) - elif lstrip:=line.strip(): - #pass - print('IGNORING:',lstrip) - - l+=1 + try: + for import_filename in import_filenames: + with open(import_filename,"rt", encoding='utf-8', errors='ignore') as f: + for line in f: + try: + if in_report: + if in_item: + if in_description: + if match := re_obj_desc_end_search(line): + item['description'].append(match.group(1)) + item['description'] = tuple(item['description']) + in_description=False + else: + item['description'].append(line) + + continue + + elif not item['description']: + if match := re_obj_desc_search(line): + item['description']=match.group(1) + continue + + elif match := re_obj_desc_begin_search(line): + in_description=True + item['description'].append(match.group(1)) + continue + + if not item['name']: + if match := re_obj_name_search(line): + item['name']=match.group(1) + continue + + if not item['ext']: + if match := re_obj_ext_search(line): + item['ext'] = match.group(1) + continue + + if not item['size']: + if match := re_obj_size_search(line): + try: + item['size']=int(match.group(1)) + except: + item['size']=0 + continue + + if not item['date']: + if match := re_obj_date_search(line): + item['date']=match.group(1) + continue + + if not item['disk_name']: + if match := re_obj_disk_name_search(line): + item['disk_name']=match.group(1) + continue + + if not item['disk_type']: + if match := re_obj_disk_type_search(line): + item['disk_type']=match.group(1) + continue + + if not item['disk_num']: + if match := re_obj_disk_num_search(line): + item['disk_num']=match.group(1) + continue + + if not item['disk_loc']: + if match := re_obj_disk_location_search(line): + item['disk_loc']=match.group(1) + continue + + if not item['path']: + if match := re_obj_path_search(line): + item['path']=match.group(1) + continue + + if not item['time']: + if match := re_obj_time_search(line): + item['time']=match.group(1) + continue + + if not item['crc']: + if match := re_obj_crc_search(line): + item['crc']=match.group(1) + continue + + if not item['category']: + if match := re_obj_category_search(line): + item['category']=match.group(1) + continue + + if not item['flag']: + if match := re_obj_flag_search(line): + item['flag']=match.group(1) + continue + + if re_obj_item_end_search(line): + in_item=False + in_description=False + + if item['type']=="Disk": + print(f'disk found :{item} -> incompatible kind of report:{import_filename}') + return [None,f'incompatible kind of report:{import_filename}'] + + elif item['type']=="File" or item['type']=="Folder": + if item['disk_name']!=demo_str and item['path']!=demo_str and item['name'] and item['name']!=demo_str and item['ext']!=demo_str and item['size']!=demo_str: + disk_name = item['disk_name'] + path = item['path'].strip('\\').split('\\') + + fileame = item['name'] + if item['ext']: + fileame+='.' + item['ext'] + + size = item['size'] + is_dir = bool(item['type']=="Folder") + is_file = bool(item['type']=="File") + is_symlink=False + is_bind=False + has_files=False + + if item['date'] and item['date']!=demo_str: + time_str=item['date'] + if item['time']!=demo_str: + time_str += ' ' + item['time'] + try: + mtime=int(mktime(parse_datetime(time_str).timetuple())) + except Exception as te: + print(f'time conv {time_str} error: {te}') + mtime=0 + else: + mtime=0 + + #scan_like_data + sld_tuple = tuple([size,is_dir,is_file,is_symlink,is_bind,has_files,mtime,item['description']]) + + if description:=item['description']: + cd_set_add(description) + cd_set_per_disk[disk_name].add(description) + + #print('description',item['description']) + ############################################################ + + path_splitted = [disk_name] + path + [fileame] + path_splitted = [path_elem for path_elem in path_splitted if path_elem] + + next_dict = wii_paths_dict + for filename in path_splitted: + filenames_set_add(filename) + + if filename not in next_dict: + next_dict[filename] = {} + next_dict = next_dict[filename] + + wii_path_tuple_to_data[tuple(path_splitted)] = sld_tuple + + ############################################################ + path_splitted = path + [fileame] + path_splitted = [path_elem for path_elem in path_splitted if path_elem] + + next_dict = wii_paths_dict_per_disk[disk_name] + for filename in path_splitted: + filenames_set_per_disk[disk_name].add(filename) + + if filename not in next_dict: + next_dict[filename] = {} + next_dict = next_dict[filename] + + wii_path_tuple_to_data_per_disk[disk_name][tuple(path_splitted)] = sld_tuple + ############################################################ + else: + pass + #print(f'another item:{item}') + + elif match := re_obj_item_search(line): + in_item=True + in_description=False + + item = { 'ext':None,'date':None,'time':None,'path':None,'name':None,'disk_name':None,'disk_type':None,'disk_loc':None,'disk_num':None,'category':None,'crc':None,'flag':None,'size':0,'description':[],'type':match.group(1)} + + elif re_obj_report_end_search(line): + in_report=False + in_item=False + in_description=False + else: + print('parse problem 1') + elif re_obj_header_search(line) or re_obj_comment_search(line): + #print(f'recognize and ignored:{line}') + pass + else: + if match := re_obj_report_search(line): + #print(f'report :"{match.group(1)}"') + in_report=True + in_item=False + in_description=False + else: + print('IGNORING:',line) + + except Exception as le: + print(f'line exception: "{line}" exception: {le}') # # $Recycle.Bin @@ -1447,11 +1604,11 @@ def get_wii_files_dict(self,import_filenames): # 0 # - return filenames_set,wii_path_tuple_to_data,wii_paths_dict - except: - return {{},{},{}} + return filenames_set,filenames_set_per_disk,wii_path_tuple_to_data,wii_path_tuple_to_data_per_disk,wii_paths_dict,wii_paths_dict_per_disk,cd_set,cd_set_per_disk + except Exception as ie: + return [None,str(ie)] - def wii_data_to_scan_like_data(self,path_list,curr_dict_ref,scan_like_data): + def wii_data_to_scan_like_data(self,path_list,curr_dict_ref,scan_like_data,customdata_helper): path_list_tuple = tuple(path_list) anything = False @@ -1466,67 +1623,115 @@ def wii_data_to_scan_like_data(self,path_list,curr_dict_ref,scan_like_data): sub_path_list_tuple = tuple(sub_path_list) try: - size,is_dir,is_file,is_symlink,is_bind,has_files,mtime = self.wii_path_tuple_to_data[sub_path_list_tuple] + size,is_dir,is_file,is_symlink,is_bind,has_files,mtime,cd = self.wii_path_tuple_to_data[sub_path_list_tuple] + if cd: + cd_field=(0,0,cd) + + try: + cd_index=customdata_helper[cd_field] + except Exception as cd_e: + print(f'{cd_e=}') + except Exception as e1: - print(f'{e1=}') - #tylko topowy ? - size=1 + #print(f'{e1=}') + #tylko topowy albo niekompletny ? + cd=None + cd_index=0 + size=0 is_dir = True is_file = False is_symlink = False is_bind = False - mtime = 4568 + mtime = 0 if is_dir: - self.wii_data_to_scan_like_data(sub_path_list,val,dict_entry) + self.wii_data_to_scan_like_data(sub_path_list,val,dict_entry,customdata_helper) if dict_entry: has_files = True else: has_files = False + if is_dir and not has_files and size>0: + is_dir=False + is_file=True + temp_list_ref = scan_like_data[name] = [size,is_dir,is_file,is_symlink,is_bind,has_files,mtime] if dict_entry: temp_list_ref.append(dict_entry) + if cd: + new_elem={} + new_elem['cd_index']=cd_index + new_elem['cd_ok']=True + + temp_list_ref.append(new_elem) + except Exception as e: print('wii_data_to_scan_like_data error:',e) - pass return anything def import_records_wii_scan(self,import_filenames): self.log.info(f'import_records_wii:{",".join(import_filenames)}') - demo_str = '*** DEMO ***' + gc_disable() + res = self.get_wii_files_dict(import_filenames) + gc_collect() + gc_enable() - quant = len(import_filenames) + if len(res)==8: + filenames_set,filenames_set_per_disk,wii_path_tuple_to_data,wii_path_tuple_to_data_per_disk,wii_paths_dict,wii_paths_dict_per_disk,cd_set,cd_set_per_disk = res - filenames_set,wii_path_tuple_to_data,wii_paths_dict = self.get_wii_files_dict(import_filenames) + quant_disks = len(wii_path_tuple_to_data_per_disk) - quant_files,quant_folders = 0,0 - for k,v in wii_path_tuple_to_data.items(): - size,is_dir,is_file,is_symlink,is_bind,has_files,mtime = v - if is_dir: - quant_folders+=1 - elif is_file: - quant_files+=1 + quant_files,quant_folders = 0,0 + for k,v in wii_path_tuple_to_data.items(): + size,is_dir,is_file,is_symlink,is_bind,has_files,mtime,cd = v + if is_dir: + quant_folders+=1 + elif is_file: + quant_files+=1 - return quant_files,quant_folders,filenames_set,wii_path_tuple_to_data,wii_paths_dict + return quant_disks,quant_files,quant_folders,filenames_set,filenames_set_per_disk,wii_path_tuple_to_data,wii_path_tuple_to_data_per_disk,wii_paths_dict,wii_paths_dict_per_disk,cd_set,cd_set_per_disk + else: + return res - def import_records_wii_do(self,quant_files,quant_folders,filenames_set,wii_path_tuple_to_data,wii_paths_dict,update_callback): + def import_records_wii_do(self,compr,postfix,label,quant_files,quant_folders,filenames_set,wii_path_tuple_to_data,wii_paths_dict,cd_set,update_callback): import_res=[] self.wii_path_tuple_to_data = wii_path_tuple_to_data self.wii_paths_dict=wii_paths_dict + new_record = self.create() + + expressions='WII_import' + use_smin=False + smin_int=0 + use_smax=False + smax_int=0 + executable='WII_import' + parameters='' + shell=False + timeout=0 + crc=False + + new_record.header.cde_list = [ [expressions,use_smin,smin_int,use_smax,smax_int,executable,parameters,shell,timeout,crc] ] + + new_record.header.scan_path = 'WII_import' + + new_record.customdata = [(0,0,cd_elem) for cd_elem in cd_set] + + customdata_helper={cd_elem_tuple:index for index,cd_elem_tuple in enumerate(new_record.customdata)} + scan_like_data={} - self.wii_data_to_scan_like_data([],self.wii_paths_dict,scan_like_data) + self.wii_data_to_scan_like_data([],self.wii_paths_dict,scan_like_data,customdata_helper) + + del customdata_helper - new_record = self.create() new_record.filenames = tuple(sorted(list(filenames_set))) - new_record.header.label = 'WII import' + new_record.header.label = label new_record.header.scan_path = 'WII import' new_record.filenames_helper = {fsname:fsname_index for fsname_index,fsname in enumerate(new_record.filenames)} @@ -1534,26 +1739,38 @@ def import_records_wii_do(self,quant_files,quant_folders,filenames_set,wii_path_ new_record.header.quant_folders = quant_folders ################################## - size,mtime = 0,0 + #size,mtime = 0,0 + mtime = 0 is_dir = True is_file = False is_symlink = False is_bind = False - has_cd = False + has_cd = bool(new_record.customdata) has_files = True cd_ok = False has_crc = False + new_record.header.references_names=0 new_record.header.references_cd=0 + sub_size,sub_quant,sub_folders_quant = new_record.sld_recalc_rec(scan_like_data) + #print('ccc',sub_size,sub_quant,sub_folders_quant,flush=True) + code = LUT_encode[ (is_dir,is_file,is_symlink,is_bind,has_cd,has_files,cd_ok,has_crc,False,False) ] - new_record.filestructure = ('',code,size,mtime,new_record.tupelize_rec(scan_like_data,print)) - new_file_path = sep.join([self.db_dir,f'wii.{int(time())}.dat']) - print(f'{new_file_path=}') + new_record.header.sum_size = sub_size + new_record.header.quant_files = sub_quant + new_record.header.quant_folders = sub_folders_quant + + new_record.header.items_names=len(new_record.filenames) + + new_record.filestructure = ('',code,sub_size,mtime,new_record.tupelize_rec(scan_like_data,print)) + + new_file_path = sep.join([self.db_dir,f'wii.{int(time())}.{postfix}.dat']) + #print(f'{new_file_path=}') - new_record.save(print,file_path=new_file_path) + new_record.save(print,file_path=new_file_path,compression_level=compr) update_callback(new_record) diff --git a/src/librer.py b/src/librer.py index 1a0d91f..3fdfbbd 100644 --- a/src/librer.py +++ b/src/librer.py @@ -472,8 +472,6 @@ def __init__(self,cwd): self.info_dialog_on_main = LabelDialog(self_main,(self.ico_librer,self.ico_librer_small),self.bg_color,pre_show=self.pre_show,post_close=self.post_close) - #self.text_ask_dialog_on_main = TextDialogQuestion(self_main,self_ico_librer,self.bg_color,pre_show=self.pre_show,post_close=self.post_close,image=self.ico_warning) - self.progress_dialog_on_load = ProgressDialog(self_main,(self.ico_librer,self.ico_librer_small),self.bg_color,pre_show=self.pre_show,post_close=self.post_close) self.progress_dialog_on_load.command_on_close = self.progress_dialog_load_abort @@ -1442,6 +1440,9 @@ def get_repack_dialog(self): self.repack_dialog_created = True return self.repack_dialog + def wii_import_dialog_name_state(self): + self.wii_import_label_entry.configure(state='disabled' if self.wii_import_separate.get() else 'normal') + wii_import_dialog_created = False @restore_status_line @block_actions_processing @@ -1471,8 +1472,7 @@ def get_wii_import_dialog(self): except: pass - (label_frame := LabelFrame(self.wii_import_dialog.area_main,text='Record Label',bd=2,bg=self.bg_color,takefocus=False)).grid(row=1,column=0,sticky='news',padx=4,pady=4,columnspan=2) - Entry(label_frame,textvariable=self.wii_import_label_var).pack(expand='yes',fill='x',padx=2,pady=2) + #(label_frame := LabelFrame(self.wii_import_dialog.area_main,text='Record Label',bd=2,bg=self.bg_color,takefocus=False)).grid(row=1,column=0,sticky='news',padx=4,pady=4,columnspan=2) (wii_import_frame := LabelFrame(self.wii_import_dialog.area_main,text='Options',bd=2,bg=self.bg_color,takefocus=False)).grid(row=2,column=0,sticky='news',padx=4,pady=4,columnspan=2) self.wii_import_dialog.area_main.grid_columnconfigure( 0, weight=1) @@ -1480,13 +1480,15 @@ def get_wii_import_dialog(self): self.wii_import_dialog.area_main.grid_rowconfigure( 2, weight=1) - self.wii_import_separate_cb = Checkbutton(wii_import_frame,text='create separate records',variable=self.wii_import_separate) - #self.wii_import_crc_cb = Checkbutton(wii_import_frame,text='Include CRC values',variable=self.wii_import_crc_var) + self.wii_import_separate_cb = Checkbutton(wii_import_frame,text=' Separate record per each disk (not recommended)',variable=self.wii_import_separate,command = self.wii_import_dialog_name_state) + self.wii_import_separate_cb.grid(row=0, column=0, sticky='wens',padx=4,pady=4,columnspan=2) - self.wii_import_separate_cb.grid(row=0, column=0, sticky='wens',padx=4,pady=4) - #self.wii_import_crc_cb.grid(row=1, column=0, sticky='wens',padx=4,pady=4) + Label(wii_import_frame,text='Common record label:',anchor='w').grid(row=1, column=0, sticky='wens',padx=4,pady=4) + self.wii_import_label_entry = Entry(wii_import_frame,textvariable=self.wii_import_label_var) + self.wii_import_label_entry.grid(row=1, column=1, sticky='wens',padx=4,pady=4) wii_import_frame.grid_columnconfigure( 0, weight=1) + wii_import_frame.grid_columnconfigure( 1, weight=1) (wii_import_frame_compr := LabelFrame(self.wii_import_dialog.area_main,text='Compression (0-22)',bd=2,bg=self.bg_color,takefocus=False)).grid(row=3,column=0,sticky='news',padx=4,pady=4,columnspan=2) @@ -1860,11 +1862,19 @@ def record_import_wii(self): initialdir = self.last_dir if self.last_dir else self.cwd self.wii_import_dialog_do_it= False if import_filenames := askopenfilenames(initialdir=self.last_dir,parent = self.main,title='Choose "Where Is It?" Report xml files to import', defaultextension=".xml",filetypes=[("XML Files","*.xml"),("All Files","*.*")]): + self.status('Parsing WII files ... ') + self.main.update() + self.last_dir = dirname(import_filenames[0]) + wiis_res = librer_core.import_records_wii_scan(import_filenames) - quant_files,quant_folders,filenames_set,wii_path_tuple_to_data,wii_paths_dict = librer_core.import_records_wii_scan(import_filenames) + if len(wiis_res)!=11: + self.info_dialog_on_main.show('Where Is It? Import failed',f"Format error.\n{wiis_res[1]}") + return - if quant_files==0 or quant_folders==0: + quant_disks,quant_files,quant_folders,filenames_set,filenames_set_per_disk,wii_path_tuple_to_data,wii_path_tuple_to_data_per_disk,wii_paths_dict,wii_paths_dict_per_disk,cd_set,cd_set_per_disk = wiis_res + + if quant_disks==0 or (quant_files==0 and quant_folders==0): self.info_dialog_on_main.show('Where Is It? Import failed',"No files / No folders") else: ########################### @@ -1873,20 +1883,51 @@ def record_import_wii(self): #self.wii_import_label_var.set(self.current_record.header.label) self.wii_import_compr_var.set(9) self.wii_import_compr_var_int.set(9) - self.wii_import_brief_label.configure(text=f'GATHERED DATA:\nfiles : {fnumber(quant_files)}\nfolders : {fnumber(quant_folders)}') + self.wii_import_brief_label.configure(text=f'GATHERED DATA:\ndisks : {fnumber(quant_disks)}\nfiles : {fnumber(quant_files)}\nfolders : {fnumber(quant_folders)}') dialog.show() + compr = self.wii_import_compr_var.get() + if self.wii_import_dialog_do_it: - res = librer_core.import_records_wii_do(quant_files,quant_folders,filenames_set,wii_path_tuple_to_data,wii_paths_dict,self.single_record_show) + postfix=0 + + if self.wii_import_separate.get(): + res= [] + #wii_path_tuple_to_data_per_disk[(disk_name,tuple(filenames_set_sd)][tuple(path_splitted)] = sld_tuple + + for disk_name,wii_path_tuple_to_data_curr in wii_path_tuple_to_data_per_disk.items(): + print(f'{disk_name=}') + self.status(f'importing {disk_name} ... ') + #sld_tuple = sub_dict[path_splitted_tuple] + quant_files=3 + quant_folders=3 + + label = disk_name + sub_res = librer_core.import_records_wii_do(compr,postfix,label,quant_files,quant_folders,filenames_set_per_disk[disk_name],wii_path_tuple_to_data_curr,wii_paths_dict_per_disk[disk_name],cd_set_per_disk[disk_name],self.single_record_show) + postfix+=1 + if sub_res: + res.append(sub_res) + + if not res: + ########################### + self.info_dialog_on_main.show('Where Is It? Import','Successful.') + self.find_clear() + else: + self.info_dialog_on_main.show('Where Is It? Import failed','\n'.join(res)) - if not res: - ########################### - self.info_dialog_on_main.show('Where Is It? Import','Successful.') - self.find_clear() else: - self.info_dialog_on_main.show('Where Is It? Import failed',res) + label = self.wii_import_label_var.get() + self.status(f'importing {label} ... ') + res = librer_core.import_records_wii_do(compr,postfix,label,quant_files,quant_folders,filenames_set,wii_path_tuple_to_data,wii_paths_dict,cd_set,self.single_record_show) + + if not res: + ########################### + self.info_dialog_on_main.show('Where Is It? Import','Successful.') + self.find_clear() + else: + self.info_dialog_on_main.show('Where Is It? Import failed',res) @restore_status_line @block_actions_processing @@ -2608,8 +2649,8 @@ def find_items(self): self_progress_dialog_on_find = self.get_progress_dialog_on_find() - gc_disable() - gc_collect() + #gc_disable() + #gc_collect() search_thread=Thread(target=lambda : librer_core.find_items_in_records(self.temp_dir,range_par, min_num,max_num, @@ -2701,8 +2742,8 @@ def find_items(self): search_thread.join self_progress_dialog_on_find.hide(True) - gc_collect() - gc_enable() + #gc_collect() + #gc_enable() find_results_quant_sum = 0 @@ -4033,6 +4074,7 @@ def show_customdata(self): self.access_customdata(record) cd_field = record.customdata[cd_index] + if cd_data := cd_field[2]: rule_nr=cd_field[0] returncode=cd_field[1] @@ -4053,7 +4095,7 @@ def show_customdata(self): self.info_dialog_on_main.show('Information','No Custom data.') except Exception as e: - self.info_dialog_on_main.show(e) + self.info_dialog_on_main.show('Custom Data Info Error',str(e)) def record_info(self): if self.actions_processing: