From e3e89eb649af41a2cc22fc79614c9a8a2ae488a4 Mon Sep 17 00:00:00 2001 From: Carl Asplund Date: Wed, 27 Feb 2013 23:38:11 +0100 Subject: [PATCH 01/15] revert temporarily to "copy non-flac files" Added option "--copyfiles" (Copy non-flac files to dest directories) When this option is selected, non-flac files present in the source directories are copied to the target directories (which, for some, is another way of handling album art). Copying only takes place when the source and target md5 hashes don't match. --- changelog.txt | 61 ---- flac2mp3.pl | 652 +++++++++++++----------------------- lib/MP3/Tag/ID3v2.pm | 6 - lib/Parallel/Forkmanager.pm | 108 ++---- 4 files changed, 267 insertions(+), 560 deletions(-) diff --git a/changelog.txt b/changelog.txt index 55bf1dc..d7fc23d 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,64 +1,3 @@ -Modified version by Carl Asplund, rev 5 (February 2013) - - Reintroduce progress reporting, this time with full support for parallel processing. The estimated - finish time is given after 20 s of transcoding, based on transcoding rate and the total file size - left to transcode. - - No longer output messages directly during transcoding or tagging of a file, but save them in a - buffer and use the callback function "run_on_finish" of Parallel::Forkmanager to display them when - the corresponding child exits. - - First perform a separate processing loop for files that don't require transcoding, while at the same time saving - the names of the files that do need transcoding. This first loop doesn't employ parallel processing, whereas the second, - transcoding loop does, through the use of the Parallel::Forkmanager module. - - Change file permissions of the tempfile/mp3-file in Tag::MP3::ID3v2 to default value (as determined by umask). - File permissions will otherwise (on *nix) be 0600 due to the odd ways of the File::Temp module (not observing the umask). - 0600 file permissions make the files invisible to some software (as is the case with Logitech Media Server). - - Turn off id3 v2.3 unsyncing - again - for increased compatibility with broken software. This time due - to inability of Logitech Media Server to cope with it. - - Update Parallel::Forkmanager to v1.0.2 - -Modified version by Carl Asplund, rev 4 (February 2013) - - Added option "--delete". When enabled, any files in the destination - directories which are not accounted for in the source directories will - be deleted before transcoding begins. In this way, the destination is kept perfectly in sync with the source, - even when files and/or directories have been renamed. - - Copying non-flac files now honors skipfiles (i.e. not copying directories with a skipfile) - - Refactored code for the translation of paths from source to target directories. - - Bugfix: Now properly suppresses all output with option "--quiet" - -Modified version by Carl Asplund, rev 3 (January 2013) - - First version which combines embedded album art, multi-CPU processing, and copying of - non-flac files present in the flac folders (which, for some, is another way of handling album art) - - Extensive refactoring of code, everything built up from latest revision in Robin - Bowes' repository: r157 - - Fixed problem with paths under MS Win systems: volume (eg. "C:") was not preserved in r157 - - Various small fixes of broken code in r157 (addressing e.g. tickets #94 and #134) - - Code for embedded album art now integrated into Robin's code for the other tags - - Using Parallel::Forkmanager instead of Proc::ParallelLoop which solves two problems: - 1. Bug in Proc::ParallelLoop casues crash for asymmeteric loads, module seems to be abandoned since 2003 - 2. Parallel::Forkmanager works under MS Win (Proc::ParallelLoop doesn't) - - Fixed race condition bug for creation of target directory (previous code broken, would sometimes skip transcoding files) - - Fixed race condition bug in MP3::Tag::ID3v2, which has severely broken temp file generation code - in original version (see https://rt.cpan.org/Public/Bug/Display.html?id=66768) - Thanks to Jason Rhinelander for posting a very useful patch on CPAN. - - Progress report and estimated-time-to-finish removed for now. Will adress this later (parallel - processing calls for another solution) - - Known problems: Parallel processing under MS Win is not perfectly stable, MP3::Tag will sometimes be unable to - rename temp files during heavy I/O operation (multiple parallel threads with lame and/or flac running), thus - skipping the file in question. However, parallel code works flawlessly on MS Win when actual transcoding - is replaced by sleep() or counting loops. Sometimes a short, 1 sec, sleep will suffice if unable to rename, - before successfully retrying rename operation. Problem seems to be unrelated to filename conflicts and - possibly goes deeper than the CPAN modules used. - -Modified version by Carl Asplund, rev 2 (November 2012) - - Non-flac files are copied to dest directory only when the source and target - md5 numbers don't match - - Progress is displayed during copying of non-flac files - - Added option "--dontcopy" (Don't copy non-flac files to dest directories) - -Modified version by Carl Asplund (August 2009) - - Embedded picture data (album art) in flac file is copied over to the mp3 file - - Estimated time to finish and progress is displayed during transcoding - - Non-flac files in the folders (jpg, mp3 files etc.) are copied over to destination folders - v0.3.0 - Remove File::Glob (it's in core) and add Text::Glob - Re-work & refactor the search/checking code diff --git a/flac2mp3.pl b/flac2mp3.pl index 4f77697..0d944b8 100755 --- a/flac2mp3.pl +++ b/flac2mp3.pl @@ -33,7 +33,6 @@ use Scalar::Util qw/ looks_like_number /; use FreezeThaw qw/ cmpStr /; use Digest::MD5; -use Time::HiRes qw ( time ); # ------- User-config options start here -------- # Assume flac and lame programs are in the path. @@ -48,21 +47,13 @@ # c:/windows/system32/flac.exe # or # c:\\windows\\system32\\flac.exe - -my ($flaccmd, $lamecmd); -if ($^O eq 'MSWin32' or $^O eq 'MSWin64') { - # This is an example of typical Windows filepaths: - $flaccmd = q|C:\Program Files (x86)\FLAC\flac.exe|; - $lamecmd = q|C:\Program Files\LAME\lame.exe|; -} -else { - $flaccmd = 'flac'; - $lamecmd = 'lame'; -} +my $flaccmd = 'flac'; +my $lamecmd = 'lame'; # Modify lame options if required my @lameargs = qw ( --noreplaygain + --vbr-new -V 2 -h --nohist @@ -162,7 +153,7 @@ GetOptions( \%Options, "quiet!", "tagdiff", "debug!", "tagsonly!", "force!", "usage", "help", "version", "pretend", "skipfile!", "skipfilename=s", - "processes=i", "tagseparator=s", "lameargs=s", "dontcopy!", "delete" + "processes=i", "tagseparator=s", "lameargs=s", "copyfiles" ); # info flag is the inverse of --quiet @@ -200,13 +191,9 @@ $cmdpath = which($cmd); } croak "$cmd not found" unless $cmdpath; - msg_info("Using $cmd from: $cmdpath"); + $Options{info} && msg("Using $cmd from: $cmdpath"); } -# Turn off unsyncing to improve compatibility with broken -# software (e.g. Logitech Media Server) -MP3::Tag->config(id3v23_unsync => 0); - # Convert directories to absolute paths $source_root = File::Spec->rel2abs($source_root); $target_root = File::Spec->rel2abs($target_root); @@ -214,273 +201,50 @@ die "Source directory not found: $source_root\n" unless -d $source_root; -msg_info( $pretendString . "Processing directory: $source_root" ); +# count all flac files in source_dir +# Display a progress report after each file, e.g. Processed 367/4394 files +# Possibly do some timing and add a Estimated Time Remaining +# Will need to only count files that are going to be processed. +# Hmmm could get complicated. + +$Options{info} + && msg( $pretendString . "Processing directory: $source_root" ); # Now look for files in the source dir # (following symlinks) -my %flac_mp3_files = get_all_paths('name', '.flac', $source_root, $target_root, '.mp3'); -my @flac_files = sort keys { %flac_mp3_files }; - -my $file_count = scalar @flac_files; -msg_info( "Found $file_count flac file" . ( $file_count > 1 ? 's' : '' . "\n" ) ); - -# If allowed, delete surplus files and folders from target directory, keeping -# it in perfect sync with (i.e. a mirror of) the source directory. -delete_excess_files_from_dest($source_root, $target_root) if ( $Options{delete} ) ; - -# Traverse all flac-files, but process only those who doesn't need transcoding. -# Save for later the name and size of each of the flac files which need transcoding. -my %flac_file_size = (); -my $total_file_size_to_transcode = 0; -my $cntr_processed = 0; -my $cntr_all; -my $t0 = 0; -foreach my $src_file (@flac_files) { - if ( $Options{force} || ( (my $result = path_and_conversion($src_file,0)) == 1 ) ) { - my $filesize = -s $src_file; - # save size data in a hash - $flac_file_size{$src_file} = $filesize; - $total_file_size_to_transcode += $filesize; - } - else { - $cntr_processed++; - if ( scalar @$result) { - msg_info(""); - foreach my $string ( @$result ) { - msg_info($string); - } - } - }; - $cntr_all ++; - - # Show the progress every second - if ( $Options{info} && - ( (!$Options{force}) || ((time - $t0) >= 1) || ($cntr_all==$file_count) ) ) { - $t0 = time; - print("\r" . $cntr_processed . " of " . $cntr_all . " flac files were processed without transcoding."); - }; - -} -msg_info(""); - - -my $files_to_trancode = scalar (keys %flac_file_size); -if ($files_to_trancode) { - my $t0 = time; # starting time for the transcoding part - my $files_transcoded_cntr = 0; - my $size_transcoded_so_far = 0; - - msg_info(""); - msg_info("The remaining $files_to_trancode files will be transcoded."); - msg_info("Total size of files to convert: " . - sprintf("%.1f", $total_file_size_to_transcode/(1024**2)) . " MB"); - # use parallel processing to launch multiple transcoding processes - msg_info("Using $Options{processes} transcoding processes.\n"); - my $pm = new Parallel::ForkManager($Options{processes}); - - $pm->run_on_finish( sub { - # This callback code is run after each transcode and outputs messages generated - # by the children, as well as overall progress info - # - # According to the Parallel::Forkmanager documentation, the structure of the - # input data @_ to this callback function is as follows: - # ($pid, $exit_code, $ident, $exit_signal, $core_dump, $data_structure_reference) - my $src = $_[2]; # this is "$ident" - my $messages = $_[5]; # this is "$data_structure_reference" - - # Update counter of transcoded files - $files_transcoded_cntr++; - - # display information about the transcoded/tagged file - foreach my $string ( @$messages ) { - msg_info($string); - } - # display updated progress info - msg_info("Processed " . - ($cntr_processed + $files_transcoded_cntr) . "/$file_count files."); - my $elapsedtime = time - $t0; - $size_transcoded_so_far += $flac_file_size{$src}; - - # After 20 seconds, we have enough data to provide a qualified guess - # about the estimated finish time. - if ($elapsedtime > 20) { - my $remains_to_be_transcoded - = $total_file_size_to_transcode - $size_transcoded_so_far; - my $transcoding_rate = $size_transcoded_so_far / $elapsedtime; - my $estim_finish_time - = localtime(time + $remains_to_be_transcoded / $transcoding_rate); - msg_info("Estimated finish time is $estim_finish_time."); - }; - - msg_info(""); - }); - - # Transcoding loop starts here - foreach my $src_file (sort keys %flac_file_size) { - $pm->start($src_file) and next; # forks here - - # transcode and generate messages with file info - my $messageref = path_and_conversion($src_file,1); - - # terminates the child process, send messages to callback sub - $pm->finish(0, $messageref ); - } - $pm->wait_all_children; - -}; - -# If allowed, copy non-flac files to destination dirs -copy_non_flacs($source_root, $target_root) unless ( $Options{dontcopy} ); - -1; -# ------------ Main program ends here -------------------------------------- - - -# ------------ Subroutines start here -------------------------------------- - -sub delete_excess_files_from_dest { - my ($source_root, $target_root) = @_; - - # Generate (source => target) hashes for the files found using - # each of the following combinations of root dirs and file suffixes - my %existing_target_mp3_files = get_all_paths('name', '.mp3', $target_root, $target_root, '.mp3'); - my %existing_source_mp3_files = get_all_paths('name', '.mp3', $source_root, $target_root, '.mp3'); - my %non_flac_files = get_all_paths('not_name', '.flac', $source_root, $target_root, ''); - my %existing_target_non_mp3_files = get_all_paths('not_name', '.mp3', $target_root, $target_root, ''); - - # 1. calculate what files to expect in directory after finished transcoding and copying - my @expected_transcoded_mp3s = keys { reverse %flac_mp3_files }; # expected mp3 files in target from transcoded flac files - my @expected_copied_files = keys { reverse %non_flac_files }; # expected files in target copied from non-flac files in source - my @expected_files = uniq(@expected_transcoded_mp3s, @expected_copied_files); # Join the arrays and remove any duplicates - - # 2. check what files are actually present - my @actual_mp3s = keys { reverse %existing_target_mp3_files }; # actual existing mp3 files in target - my @actual_non_mp3s = keys { reverse %existing_target_non_mp3_files }; # existing non-mp3 files in target - my @actual_files = (@actual_mp3s, @actual_non_mp3s); # Join the arrays (being mutually exclusive, there is no overlap) - - # 3. determine which files to remove from target directory tree - my @files_to_remove = single_difference(\@expected_files, \@actual_files); - - # 4. determine which subdirectories to remove from target directory tree - my @expected_subdirs_in_target = get_all_dirs($source_root,$target_root); - my @actual_subdirs_in_target = get_all_dirs($target_root,$target_root); - my @dirs_to_remove = single_difference(\@expected_subdirs_in_target, \@actual_subdirs_in_target); - - # 5. carry out the deletions - foreach my $file (@files_to_remove) { - $Options{pretend} || unlink $file or die "Unable to delete $file: $!"; - msg_info($pretendString . "Deleted \"$file\""); - } - foreach my $dir (reverse sort @dirs_to_remove) { - $Options{pretend} || File::Path->remove_tree($dir) or die "Unable to delete directory $dir: $!"; - msg_info($pretendString . "Deleted directory \"$dir\""); - } -} - -# Return all unique elements of input array @_ -sub uniq { - return sort keys %{{ map { $_ => 1 } @_ }} -}; -# Acccept two arrays @A and @B as argument, return elements in @B that aren't in @A. -sub single_difference { - my ($A, $B) = @_; - - # build lookup table - my %seen = (); - my @bonly = (); - @seen{@$A} = (1) x @$A; - foreach my $item (@$B) { - push(@bonly, $item) unless $seen{$item}; - } - return sort @bonly; -} +my @flac_files = @{ find_files( $source_root, qr/\.flac$/i ) }; -sub get_all_dirs { - my ($root, $new_root) = @_; - # we supply no suffix, so we search for directories (not files): - my @orig_dirs = @{ find_files_or_dirs($root) }; - - my @dirs = (); - foreach my $dir (@orig_dirs) { - # strip source root dir from path... - my $rel_path = File::Spec->abs2rel( $dir, $root ); - # then replace it with target root dir - push @dirs, File::Spec->rel2abs( $rel_path, $new_root ); - } - return sort @dirs; -} +# Get directories from target_dir and put in an array +my ( $target_root_volume, $target_root_path, $target_root_file ) = File::Spec->splitpath( $target_root, 1 ); +my @target_root_elements = File::Spec->splitdir($target_root_path); -sub get_all_paths { - my ($rule, $suffix, $root, $new_root, $new_suffix) = @_; - my @orig_files = @{ find_files_or_dirs($root, $rule, $suffix) }; - - # Even if $root = $new_root, we need to do the following operations - # to get a consistent path format (otherwise problematic in e.g. MS Win) - # that is suitable for later string comparison: - my %paths = (); - foreach my $src (@orig_files) { - # Strip source root dir from file path - my $rel_path = File::Spec->abs2rel( $src, $root ); - # ... then replace it with target root dir and change file suffix. - ($paths{$src} = File::Spec->rel2abs( $rel_path, $new_root ) ) =~ s{$suffix$}{$new_suffix}xmsi; - } - return %paths +# use parallel processing to launch multiple transcoding processes +msg("Using $Options{processes} transcoding processes.\n"); +my $pm = new Parallel::ForkManager($Options{processes}); +foreach my $src_file (@flac_files) { + $pm->start and next; # Forks and returns the pid for the child + path_and_conversion($src_file); + $pm->finish; # Terminates the child process } +$pm->wait_all_children; -sub find_files_or_dirs { - my $path = shift; - my $rule = shift; - my $suffix = shift; - - # If a matching rule and file suffix is defined we are looking for files, - # otherwise we are looking for directories. - my $found_list; - if (defined $rule && defined $suffix) { - $found_list = File::Find::Rule->file()->extras( { follow => 1 } )->$rule(qr{$suffix$}xmsi) - } - else { - $found_list = File::Find::Rule->directory->extras( { follow => 1 } ) - }; - - # skip any directories where a "skipfile" is found - my @found; - if ( $Options{skipfile} && ($path eq $source_root) ) { - my $skip_list = File::Find::Rule->directory->exec( - sub { - my ( $fname, $fpath, $frpath ) = @_; - if ( -f File::Spec->catdir( $frpath, $Options{skipfilename} ) ) { - return 1; - } - else { - return 0; - } - } - )->prune->discard; - @found = sort File::Find::Rule->or( $skip_list, $found_list )->in($path); - } - else { - @found = sort $found_list ->in($path); - } -return \@found; -} -sub copy_non_flacs { - my ($source_root, $target_root) = @_; - - my %non_flac_files = get_all_paths('not_name', '.flac', $source_root, $target_root, ''); - my @non_flac_files = keys %non_flac_files; +if ( $Options{copyfiles} ) { + my @non_flac_files + = sort File::Find::Rule->file()->extras( { follow => 1 } )->not_name(qr/\.flac$/i) + ->in($source_root); my $non_flac_file_count = scalar @non_flac_files; - msg_info( "Found $non_flac_file_count non-flac file" . - ( $non_flac_file_count != 1 ? 's' : '' . "\n" ) ); + $Options{info} && + msg( "Found $non_flac_file_count non-flac file" .( $non_flac_file_count != 1 ? 's' : '' . "\n" ) ); # Copy non-flac files from source to dest directories my $t0 = time; my $cntr_all = 0; my $cntr_copied = 0; foreach my $src_file (@non_flac_files) { - my $dst_file = $non_flac_files{$src_file}; - # Flag which determines if file should be copied: + my ($dst_dir, $dst_file) = get_dest_file_path_non_flac($src_file); + # Flag which determines if file should be copied: my $do_copy = 1; # Don't copy file if it already exists in dest directory and # has identical md5 to the source file @@ -492,13 +256,11 @@ sub copy_non_flacs { }; } else { - # Create the destination directory if it - # doesn't already exist - (undef, my $dst_dir) = - File::Basename::fileparse($dst_file); # retrieve directory name - unless ( $Options{pretend} || -d $dst_dir ) { - mkpath($dst_dir) or die "Can't create directory $dst_dir\n"; - } + # Create the destination directory if it + # doesn't already exist + mkpath($dst_dir) + or die "Can't create directory $dst_dir\n" + unless -d $dst_dir; }; if ( $do_copy ) { unless ( $Options{pretend} ) { @@ -508,14 +270,35 @@ sub copy_non_flacs { }; $cntr_all ++; # Show the progress every second - if ( $Options{info} && - ( ((time - $t0) >= 1) || ($cntr_all==$non_flac_file_count) ) ) { + if ( ((time - $t0) >= 1) || ($cntr_all==$non_flac_file_count) ) { $t0 = time; - print("\r" . $pretendString . $cntr_copied . - " non-flac files of " . $cntr_all ." were copied to dest directories."); + print("\r" . $pretendString . $cntr_copied . " non-flac files of " . $cntr_all ." were copied to dest directories."); }; }; - msg_info("\n"); # double line feed + msg("\n"); # double line feed +}; + + +sub get_dest_file_path_non_flac { + my $source = shift; + + # remove $source_dir from front of $src_file + my $target = $source; + $target =~ s{\Q$source_root/\E}{}xms; + + # Get directories in target and put in an array + # Note: the filename is the source file name + my ( $target_volume, $target_path, $source_file ) = File::Spec->splitpath($target); + my @target_path_elements = File::Spec->splitdir($target_path); + + # Add the dst_dirs to the dst root and join back together + $target_path = File::Spec->catdir( @target_root_elements, @target_path_elements ); + + # Now join it all together to get the complete path of the dest_file + $target = File::Spec->catpath( $target_root_volume, $target_path, $source_file ); + my $target_dir = File::Spec->catpath( $target_root_volume, $target_path, '' ); + + return $target_dir,$target; }; sub get_md5_of_non_flac_file { @@ -527,95 +310,120 @@ sub get_md5_of_non_flac_file { return $md5_code; }; +# use parallel processing to launch multiple transcoding processes sub path_and_conversion{ - # When "$transcode_enabled" is false it means that we are - # checking whether the file should be transcoded. If so, we - # return, and no further processing is taking place during this call. - # If the file is not to be transcoded, we stay and update tags if - # necessary. - - my $source = shift; - my $target = $flac_mp3_files{$source}; - my $transcode_enabled = shift; - my @messages = (); - my $t0; - my $elapsed_time = undef; - - $Options{debug} && msg("source: '$source'"); - $Options{debug} && msg("target: '$target'"); + my $source = shift; - # Step 1: get tags from flac file - my $source_tags = read_flac_tags($source); + # remove $source_dir from front of $src_file + my $target = $source; + $target =~ s{\Q$source_root/\E}{}xms; - # Step 2: hash to hold tags that will be updated - my $tags_to_update = preprocess_flac_tags( $source_tags ); + # Get directories in target and put in an array + # Note: the filename is the source file name + my ( $target_volume, $target_path, $source_file ) = File::Spec->splitpath($target); + my @target_path_elements = File::Spec->splitdir($target_path); - # Step 3: Initialise file processing flags - my ($pflags, $mess) = examine_destfile_tags( $target, $tags_to_update ); - push @messages, @$mess; + # Add the dst_dirs to the dst root and join back together + $target_path = File::Spec->catdir( @target_root_elements, @target_path_elements ); + # Add volume for OSes that require it (MSWin etc.) + $target_path = File::Spec->catpath( $target_root_volume, $target_path, '' ); - if ( ( !$$pflags{exists} || $$pflags{md5} || $Options{force} ) - && !$Options{tagsonly} ) { - - # Return if this file would be transcoded - return 1 unless ($transcode_enabled); - - $t0 = time; - # Step 4: Transcode the file based on the processing flags - $mess = transcode_file( $source, $target, $pflags ); - push @messages, @$mess; - $elapsed_time = time - $t0; - }; + # Get the basename of the dst file + my ( $target_base, $target_dir, $source_ext ) = fileparse( $source_file, qr{\Q.flac\E$}xmsi ); - # Step 5: Write the tags based on the processing flags - $mess = write_tags( $target, $tags_to_update, $pflags ); - push @messages, @$mess; + # Now join it all together to get the complete path of the dest_file + $target = File::Spec->catpath( $target_volume, $target_path, $target_base . '.mp3' ); - if (defined $elapsed_time) { - push @messages, sprintf("Conversion took %5.2f seconds.", $elapsed_time); - }; - - return \@messages; + convert_file( $source, $target ); }; -sub showusage { - print <<"EOT"; +1; + +sub find_files { + my $path = shift; + my $regex = shift; -Usage: $0 [--pretend] [--quiet] [--debug] [--tagsonly] [--force] [--tagdiff] [--noskipfile] [--skipfilename=] [--lameargs='parameter-list'] + my @found_files; - --pretend Don't actually do anything + my $found_list = File::Find::Rule->extras( { follow => 1 } )->name($regex); + if ( $Options{skipfile} ) { + my $skip_list = File::Find::Rule->directory->exec( + sub { + my ( $fname, $fpath, $frpath ) = @_; + if ( -f File::Spec->catdir( $frpath, $Options{skipfilename} ) ) { + return 1; + } + else { + return 0; + } + } + )->prune->discard; + @found_files = sort File::Find::Rule->or( $skip_list, $found_list )->in($path); + } + else { + + @found_files = sort $found_list ->in($path); + } + + $Options{debug} && msg( Dumper(@found_files) ); + + if ( $Options{info} ) { + my $file_count = scalar @found_files; + msg( "Found $file_count flac file" . ( $file_count > 1 ? 's' : '' . "\n" ) ); + } + + return \@found_files; +} + +sub showusage { + print <<"EOT"; +Usage: $0 [--pretend] [--quiet] [--debug] [--tagsonly] [--force] [--tagdiff] [--noskipfile] [--skipfilename=] [--lameargs='parameter-list'] --pretend Don't actually do anything --quiet Disable informational output to stdout --debug Enable debugging output. For developers only! --tagsonly Don't do any transcoding - just update tags --force Force transcoding and tag update even if not required --tagdiff Print source/dest tag values if different --lameargs='s' specify parameter(string) to be passed to the LAME Encoder - Default: "--noreplaygain -V 2 -h --nohist --quiet" + Default: "--noreplaygain --vbr-new -V 2 -h --nohist --quiet" --noskipfile Ignore any skip files --skipfilename Specify the name of the skip file. Default: flac2mp3.ignore - --processes=n Launch n parallel transcoding processes (limited support on Windows platform) + --processes=n Launch n parallel transcoding processes (does not work on Windows platform) Use with multi-core CPUs. Default: 1 --tagseparator=s Use "s" as the separator to join multiple instances of the same tag. Default: "/" - --dontcopy Don't copy non-flac files to dest directories - --delete Delete surplus files and directories in destination, keeping in sync with source dir - -Example: $0 --processes=6 --delete ~/FLAC/ ~/MP3/ - + --copyfiles Copy non-flac files to dest directories EOT exit 0; } -sub msg { - print "@_\n" +sub msg { + my $msg = shift; + print "$msg\n"; } -sub msg_info { - # display only if "--quiet" option is not in use - $Options{info} && msg(@_) +sub convert_file { + my ( $source, $target ) = @_; + + $Options{debug} && msg("source: '$source'"); + $Options{debug} && msg("target: '$target'"); + + # get tags from flac file + my $source_tags = read_flac_tags($source); + + # hash to hold tags that will be updated + my $tags_to_update = preprocess_flac_tags( $source_tags ); + + # Initialise file processing flags + my $pflags = examine_destfile_tags( $target, $tags_to_update ); + + # Transcode the file based on the processing flags + transcode_file( $source, $target, $pflags ); + + # Write the tags based on the processing flags + write_tags( $target, $tags_to_update, $pflags ); } sub read_flac_tags { @@ -704,7 +512,6 @@ sub examine_destfile_tags { my $destfilename = shift; my $frames_ref = shift; my %frames_to_update = %$frames_ref; # this is only to minimize changes - my @return_messages = (); # Initialise file processing flags my %pflags = ( @@ -814,12 +621,11 @@ sub examine_destfile_tags { if ( $dest_text ne $srcframe ) { $pflags{tags} = 1; if ( $Options{tagdiff} ) { - push @return_messages, ( - "frame: '$frame'", - "srcframe value: '$srcframe'", - "destframe value: '$dest_text'" - ); + msg("frame: '$frame'"); + msg("srcframe value: '$srcframe'"); + msg("destframe value: '$dest_text'"); } + } } } @@ -841,7 +647,7 @@ sub examine_destfile_tags { msg( Dumper \%frames_to_update ); } - return \%pflags, \@return_messages; + return \%pflags; } sub transcode_file { @@ -849,92 +655,93 @@ sub transcode_file { my $target = shift; my $pflags_ref = shift; my %pflags = %$pflags_ref; # this is only to minimize changes - my @return_messages = (); - my $elapsed_time; - - # Transcode to a temp file in the destdir. - # Rename the file if the conversion completes sucessfully - # This avoids leaving incomplete files in the destdir - # If we're "pretending", don't create a File::Temp object - my $tmpfilename; - my $tmpfh; - if ( $Options{pretend} ) { - $tmpfilename = $target; - } - else { - # retrieve destination directory name - (undef, my $dst_dir) = File::Basename::fileparse($target); - # Create the destination directory if it - # doesn't already exist - unless (-d $dst_dir) { - # If necessary, allow a second check. Don't die just because the - # dir was created by another child (race condition): - mkpath($dst_dir) or (-d $dst_dir) - or die "Can't create directory $dst_dir\n"; - }; - $tmpfh = new File::Temp( - UNLINK => 1, - DIR => $dst_dir, - SUFFIX => '.tmp' - ); - $tmpfilename = $tmpfh->filename; - } - # Save message to be displayed on screen - push @return_messages, $pretendString . "Transcoding \"$source\"" ; - - my $convert_command = - "\"$flaccmd\" @flacargs \"$source\"" . "| \"$lamecmd\" @lameargs - \"$tmpfilename\""; + my ( $target_volume, $target_dir, $target_filename ) = File::Spec->splitpath($target); + my $dst_dir = File::Spec->catpath( $target_volume, $target_dir, '' ); + + if ( ( !$pflags{exists} || $pflags{md5} || $Options{force} ) + && !$Options{tagsonly} ) + { + + # Transcode to a temp file in the destdir. + # Rename the file if the conversion completes sucessfully + # This avoids leaving incomplete files in the destdir + # If we're "pretending", don't create a File::Temp object + my $tmpfilename; + my $tmpfh; + if ( $Options{pretend} ) { + $tmpfilename = $target; + } + else { + + # Create the destination directory if it + # doesn't already exist + unless (-d $dst_dir) { + # If necessary, allow a second check. Don't die just because the + # dir was created by another child (race condition): + mkpath($dst_dir) or (-d $dst_dir) + or die "Can't create directory $dst_dir\n"; + }; + $tmpfh = new File::Temp( + UNLINK => 1, + DIR => $dst_dir, + SUFFIX => '.tmp' + ); + $tmpfilename = $tmpfh->filename; + } + $Options{info} + && msg( $pretendString . "Transcoding \"$source\"" ); + + my $convert_command = "\"$flaccmd\" @flacargs \"$source\"" . "| \"$lamecmd\" @lameargs - \"$tmpfilename\""; - $Options{debug} && msg("transcode: $convert_command"); + $Options{debug} && msg("transcode: $convert_command"); - # Convert the file (unless we're pretending} - my $exit_value; - if ( !$Options{pretend} ) { - $exit_value = system($convert_command); - } - else { - $exit_value = 0; - } + # Convert the file (unless we're pretending} + my $exit_value; + if ( !$Options{pretend} ) { + $exit_value = system($convert_command); + } + else { + $exit_value = 0; + } - $Options{debug} - && msg("Exit value from convert command: $exit_value"); + $Options{debug} + && msg("Exit value from convert command: $exit_value"); - if ($exit_value) { - msg("$convert_command failed with exit code $exit_value"); + if ($exit_value) { + msg("$convert_command failed with exit code $exit_value"); - # delete the destfile if it exists - unlink $tmpfilename; + # delete the destfile if it exists + unlink $tmpfilename; - # should check exit status of this command - exit($exit_value); - } + # should check exit status of this command - if ( !$Options{pretend} ) { + exit($exit_value); + } - # If we get here, assume the conversion has succeeded - $tmpfh->unlink_on_destroy(0); - $tmpfh->close; - croak "Failed to rename '$tmpfilename' to '$target' $!" - unless rename( $tmpfilename, $target ); + if ( !$Options{pretend} ) { - # the destfile now exists! - $pflags{exists} = 1; + # If we get here, assume the conversion has succeeded + $tmpfh->unlink_on_destroy(0); + $tmpfh->close; + croak "Failed to rename '$tmpfilename' to '$target' $!" + unless rename( $tmpfilename, $target ); - # and the tags need writing - $pflags{tags} = 1; - } + # the destfile now exists! + $pflags{exists} = 1; + # and the tags need writing + $pflags{tags} = 1; + } + } if ( $Options{debug} ) { msg("pf_exists: $pflags{exists}"); msg("pf_tags: $pflags{tags}"); msg( "\$Options{pretend}: " . ( $Options{pretend} ? 'set' : 'not set' ) ); - } - + } + %$pflags_ref = %pflags; # this is only to minimize changes - - return \@return_messages; } sub write_tags { @@ -943,13 +750,16 @@ sub write_tags { my $pflags_ref = shift; my %frames_to_update = %$frames_ref; # this is only to minimize changes my %pflags = %$pflags_ref; # this is only to minimize changes - my @return_messages = (); # Write the tags - if ( $pflags{exists} && ( $pflags{tags} || $Options{force} ) ) { + if ($pflags{exists} + && ( $pflags{tags} + || $Options{force} ) + ) + { - # save message to be displayed on screen - push @return_messages, $pretendString . "Writing tags to \"$destfilename\""; + $Options{info} + && msg( $pretendString . "Writing tags to \"$destfilename\"" ); if ( !$Options{pretend} ) { my $mp3 = MP3::Tag->new($destfilename); @@ -1012,7 +822,6 @@ sub write_tags { # utime $srcstat->mtime, $srcstat->mtime, $destfilename; } } - return \@return_messages; } sub INT_Handler { @@ -1038,7 +847,8 @@ sub fixUpTrackNumber { $trackNum = sprintf( "%02u", $trackNum ); } else { - msg_info('TRACKNUMBER not numeric'); + $Options{info} + && msg('TRACKNUMBER not numeric'); } } return $trackNum; @@ -1088,4 +898,4 @@ sub picsToAPICframes { # vim:set softtabstop=4: # vim:set shiftwidth=4: -__END__ \ No newline at end of file +__END__ diff --git a/lib/MP3/Tag/ID3v2.pm b/lib/MP3/Tag/ID3v2.pm index e7e3934..273d625 100755 --- a/lib/MP3/Tag/ID3v2.pm +++ b/lib/MP3/Tag/ID3v2.pm @@ -636,12 +636,6 @@ sub insert_space { ); my $tempfile = $tempfh->filename; - # Change file permissions to default value. - # Permissions will otherwise (at least on *nix) - # be 0600 regardless of umask, due a peculiar choice choice - # by the developers of the File::Temp module. - chmod umask ^ 0666, $tempfile; - if ($@) { warn "Can't open '$tempfile' to insert tag\n"; return -1; diff --git a/lib/Parallel/Forkmanager.pm b/lib/Parallel/Forkmanager.pm index 29471a8..c3d9491 100644 --- a/lib/Parallel/Forkmanager.pm +++ b/lib/Parallel/Forkmanager.pm @@ -6,7 +6,7 @@ Parallel::ForkManager - A simple parallel processing fork manager use Parallel::ForkManager; - $pm = Parallel::ForkManager->new($MAX_PROCESSES); + $pm = new Parallel::ForkManager($MAX_PROCESSES); foreach $data (@all_data) { # Forks and returns the pid for the child: @@ -39,7 +39,7 @@ The code for a downloader would look something like this: ... # Max 30 processes for parallel download - my $pm = Parallel::ForkManager->new(30); + my $pm = new Parallel::ForkManager(30); foreach my $linkarray (@links) { $pm->start and next; # do the fork @@ -83,11 +83,11 @@ will be forked. This is intended for debugging purposes. The optional second parameter, $tempdir, is only used if you want the children to send back a reference to some data (see RETRIEVING DATASTRUCTURES -below). If not provided, it is set to $L->tmpdir(). The new method will die if the temporary directory does not exist or it is not a directory, whether you provided this parameter or the -$L->tmpdir() is used. =item start [ $process_identifier ] # P @@ -210,7 +210,7 @@ This small example can be used to get URLs in parallel. use Parallel::ForkManager; use LWP::Simple; - my $pm= Parallel::ForkManager->new(10); + my $pm=new Parallel::ForkManager(10); for my $link (@ARGV) { $pm->start and next; my ($fn)= $link =~ /^.*\/(.*?)$/; @@ -236,22 +236,25 @@ Example of a program using callbacks to get child exit codes: my @names = qw( Fred Jim Lily Steve Jessica Bob Dave Christine Rico Sara ); # hash to resolve PID's back to child specific information - my $pm = Parallel::ForkManager->new($max_procs); + my $pm = new Parallel::ForkManager($max_procs); # Setup a callback for when a child finishes up so we can # get it's exit code - $pm->run_on_finish( sub { - my ($pid, $exit_code, $ident) = @_; + $pm->run_on_finish( + sub { my ($pid, $exit_code, $ident) = @_; print "** $ident just got out of the pool ". "with PID $pid and exit code: $exit_code\n"; - }); + } + ); - $pm->run_on_start( sub { - my ($pid,$ident)=@_; + $pm->run_on_start( + sub { my ($pid,$ident)=@_; print "** $ident started, pid: $pid\n"; - }); + } + ); - $pm->run_on_wait( sub { + $pm->run_on_wait( + sub { print "** Have to wait for one children ...\n" }, 0.5 @@ -279,7 +282,7 @@ In this simple example, each child sends back a string reference. use Parallel::ForkManager 0.7.6; use strict; - my $pm = Parallel::ForkManager->new(2, '/server/path/to/temp/dir/'); + my $pm = new Parallel::ForkManager(2, '/server/path/to/temp/dir/'); # data structure retrieval and handling $pm -> run_on_finish ( # called BEFORE the first call to start() @@ -323,7 +326,7 @@ process whatever is retrieved. use Data::Dumper; # to display the data structures retrieved. use strict; - my $pm = Parallel::ForkManager->new(20); # using the system temp dir $L->tmpdir() # data structure retrieval and handling my %retrieved_responses = (); # for collecting responses @@ -435,16 +438,12 @@ package Parallel::ForkManager; use POSIX ":sys_wait_h"; use Storable qw(store retrieve); use File::Spec; -use File::Temp (); -use File::Path (); use strict; use vars qw($VERSION); -$VERSION="1.02"; +$VERSION="0.7.9"; $VERSION = eval $VERSION; -sub new { - my ($c,$processes, $tempdir)=@_; - +sub new { my ($c,$processes, $tempdir)=@_; my $h={ max_proc => $processes, processes => {}, @@ -453,17 +452,14 @@ sub new { }; # determine temporary directory for storing data structures - # add it to Parallel::ForkManager object so children can use it - # We don't let it clean up so it won't do it in the child process - # but we have our own DESTROY to do that. - $h->{tempdir} = File::Temp::tempdir(CLEANUP => 0); + $tempdir = File::Spec->tmpdir() unless (defined($tempdir) && length($tempdir)); + die qq|Temporary directory "$tempdir" doesn't exist or is not a directory.| unless (-e $tempdir && -d _); # ensure temp dir exists and is indeed a directory + $h->{tempdir} = $tempdir; # add tempdir to Parallel::ForkManager object so children can use it return bless($h,ref($c)||$c); }; -sub start { - my ($s,$identification)=@_; - +sub start { my ($s,$identification)=@_; die "Cannot start another process while you are in the child process" if $s->{in_child}; while ($s->{max_proc} && ( keys %{ $s->{processes} } ) >= $s->{max_proc}) { @@ -488,9 +484,7 @@ sub start { } } -sub finish { - my ($s, $x, $r)=@_; - +sub finish { my ($s, $x, $r)=@_; if ( $s->{in_child} ) { if (defined($r)) { # store the child's data structure my $storable_tempfile = File::Spec->catfile($s->{tempdir}, 'Parallel-ForkManager-' . $s->{parent_pid} . '-' . $$ . '.txt'); @@ -510,9 +504,7 @@ sub finish { return 0; } -sub wait_children { - my ($s)=@_; - +sub wait_children { my ($s)=@_; return if !keys %{$s->{processes}}; my $kid; do { @@ -522,9 +514,7 @@ sub wait_children { *wait_childs=*wait_children; # compatibility -sub wait_one_child { - my ($s,$par)=@_; - +sub wait_one_child { my ($s,$par)=@_; my $kid; while (1) { $kid = $s->_waitpid(-1,$par||=0); @@ -553,9 +543,7 @@ sub wait_one_child { $kid; }; -sub wait_all_children { - my ($s)=@_; - +sub wait_all_children { my ($s)=@_; while (keys %{ $s->{processes} }) { $s->on_wait; $s->wait_one_child(defined $s->{on_wait_period} ? &WNOHANG : undef); @@ -564,29 +552,21 @@ sub wait_all_children { *wait_all_childs=*wait_all_children; # compatibility; -sub run_on_finish { - my ($s,$code,$pid)=@_; - +sub run_on_finish { my ($s,$code,$pid)=@_; $s->{on_finish}->{$pid || 0}=$code; } -sub on_finish { - my ($s,$pid,@par)=@_; - +sub on_finish { my ($s,$pid,@par)=@_; my $code=$s->{on_finish}->{$pid} || $s->{on_finish}->{0} or return 0; $code->($pid,@par); }; -sub run_on_wait { - my ($s,$code, $period)=@_; - +sub run_on_wait { my ($s,$code, $period)=@_; $s->{on_wait}=$code; $s->{on_wait_period} = $period; } -sub on_wait { - my ($s)=@_; - +sub on_wait { my ($s)=@_; if(ref($s->{on_wait}) eq 'CODE') { $s->{on_wait}->(); if (defined $s->{on_wait_period}) { @@ -596,21 +576,15 @@ sub on_wait { }; }; -sub run_on_start { - my ($s,$code)=@_; - +sub run_on_start { my ($s,$code)=@_; $s->{on_start}=$code; } -sub on_start { - my ($s,@par)=@_; - +sub on_start { my ($s,@par)=@_; $s->{on_start}->(@par) if ref($s->{on_start}) eq 'CODE'; }; -sub set_max_procs { - my ($s, $mp)=@_; - +sub set_max_procs { my ($s, $mp)=@_; $s->{max_proc} = $mp; } @@ -622,9 +596,7 @@ sub _waitpid { # Call waitpid() in the standard Unix fashion. # On ActiveState Perl 5.6/Win32 build 625, waitpid(-1, &WNOHANG) always # blocks unless an actual PID other than -1 is given. -sub _NT_waitpid { - my ($s, $pid, $par) = @_; - +sub _NT_waitpid { my ($s, $pid, $par) = @_; if ($par == &WNOHANG) { # Need to nonblock on each of our PIDs in the pool. my @pids = keys %{ $s->{processes} }; # Simulate -1 (no processes awaiting cleanup.) @@ -648,12 +620,4 @@ sub _NT_waitpid { } } -sub DESTROY { - my ($self) = @_; - - if ($self->{parent_pid} == $$ && -d $self->{tempdir}) { - File::Path::remove_tree($self->{tempdir}); - } -} - 1; From d0a6c07d16df439d92b8dedc57be720c3b28cd71 Mon Sep 17 00:00:00 2001 From: Carl Asplund Date: Thu, 28 Feb 2013 00:41:46 +0100 Subject: [PATCH 02/15] Revert "revert temporarily to "copy non-flac files"" This reverts commit e3e89eb649af41a2cc22fc79614c9a8a2ae488a4. --- changelog.txt | 61 ++++ flac2mp3.pl | 652 +++++++++++++++++++++++------------- lib/MP3/Tag/ID3v2.pm | 6 + lib/Parallel/Forkmanager.pm | 108 ++++-- 4 files changed, 560 insertions(+), 267 deletions(-) diff --git a/changelog.txt b/changelog.txt index d7fc23d..55bf1dc 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,64 @@ +Modified version by Carl Asplund, rev 5 (February 2013) + - Reintroduce progress reporting, this time with full support for parallel processing. The estimated + finish time is given after 20 s of transcoding, based on transcoding rate and the total file size + left to transcode. + - No longer output messages directly during transcoding or tagging of a file, but save them in a + buffer and use the callback function "run_on_finish" of Parallel::Forkmanager to display them when + the corresponding child exits. + - First perform a separate processing loop for files that don't require transcoding, while at the same time saving + the names of the files that do need transcoding. This first loop doesn't employ parallel processing, whereas the second, + transcoding loop does, through the use of the Parallel::Forkmanager module. + - Change file permissions of the tempfile/mp3-file in Tag::MP3::ID3v2 to default value (as determined by umask). + File permissions will otherwise (on *nix) be 0600 due to the odd ways of the File::Temp module (not observing the umask). + 0600 file permissions make the files invisible to some software (as is the case with Logitech Media Server). + - Turn off id3 v2.3 unsyncing - again - for increased compatibility with broken software. This time due + to inability of Logitech Media Server to cope with it. + - Update Parallel::Forkmanager to v1.0.2 + +Modified version by Carl Asplund, rev 4 (February 2013) + - Added option "--delete". When enabled, any files in the destination + directories which are not accounted for in the source directories will + be deleted before transcoding begins. In this way, the destination is kept perfectly in sync with the source, + even when files and/or directories have been renamed. + - Copying non-flac files now honors skipfiles (i.e. not copying directories with a skipfile) + - Refactored code for the translation of paths from source to target directories. + - Bugfix: Now properly suppresses all output with option "--quiet" + +Modified version by Carl Asplund, rev 3 (January 2013) + - First version which combines embedded album art, multi-CPU processing, and copying of + non-flac files present in the flac folders (which, for some, is another way of handling album art) + - Extensive refactoring of code, everything built up from latest revision in Robin + Bowes' repository: r157 + - Fixed problem with paths under MS Win systems: volume (eg. "C:") was not preserved in r157 + - Various small fixes of broken code in r157 (addressing e.g. tickets #94 and #134) + - Code for embedded album art now integrated into Robin's code for the other tags + - Using Parallel::Forkmanager instead of Proc::ParallelLoop which solves two problems: + 1. Bug in Proc::ParallelLoop casues crash for asymmeteric loads, module seems to be abandoned since 2003 + 2. Parallel::Forkmanager works under MS Win (Proc::ParallelLoop doesn't) + - Fixed race condition bug for creation of target directory (previous code broken, would sometimes skip transcoding files) + - Fixed race condition bug in MP3::Tag::ID3v2, which has severely broken temp file generation code + in original version (see https://rt.cpan.org/Public/Bug/Display.html?id=66768) + Thanks to Jason Rhinelander for posting a very useful patch on CPAN. + - Progress report and estimated-time-to-finish removed for now. Will adress this later (parallel + processing calls for another solution) + - Known problems: Parallel processing under MS Win is not perfectly stable, MP3::Tag will sometimes be unable to + rename temp files during heavy I/O operation (multiple parallel threads with lame and/or flac running), thus + skipping the file in question. However, parallel code works flawlessly on MS Win when actual transcoding + is replaced by sleep() or counting loops. Sometimes a short, 1 sec, sleep will suffice if unable to rename, + before successfully retrying rename operation. Problem seems to be unrelated to filename conflicts and + possibly goes deeper than the CPAN modules used. + +Modified version by Carl Asplund, rev 2 (November 2012) + - Non-flac files are copied to dest directory only when the source and target + md5 numbers don't match + - Progress is displayed during copying of non-flac files + - Added option "--dontcopy" (Don't copy non-flac files to dest directories) + +Modified version by Carl Asplund (August 2009) + - Embedded picture data (album art) in flac file is copied over to the mp3 file + - Estimated time to finish and progress is displayed during transcoding + - Non-flac files in the folders (jpg, mp3 files etc.) are copied over to destination folders + v0.3.0 - Remove File::Glob (it's in core) and add Text::Glob - Re-work & refactor the search/checking code diff --git a/flac2mp3.pl b/flac2mp3.pl index 0d944b8..4f77697 100755 --- a/flac2mp3.pl +++ b/flac2mp3.pl @@ -33,6 +33,7 @@ use Scalar::Util qw/ looks_like_number /; use FreezeThaw qw/ cmpStr /; use Digest::MD5; +use Time::HiRes qw ( time ); # ------- User-config options start here -------- # Assume flac and lame programs are in the path. @@ -47,13 +48,21 @@ # c:/windows/system32/flac.exe # or # c:\\windows\\system32\\flac.exe -my $flaccmd = 'flac'; -my $lamecmd = 'lame'; + +my ($flaccmd, $lamecmd); +if ($^O eq 'MSWin32' or $^O eq 'MSWin64') { + # This is an example of typical Windows filepaths: + $flaccmd = q|C:\Program Files (x86)\FLAC\flac.exe|; + $lamecmd = q|C:\Program Files\LAME\lame.exe|; +} +else { + $flaccmd = 'flac'; + $lamecmd = 'lame'; +} # Modify lame options if required my @lameargs = qw ( --noreplaygain - --vbr-new -V 2 -h --nohist @@ -153,7 +162,7 @@ GetOptions( \%Options, "quiet!", "tagdiff", "debug!", "tagsonly!", "force!", "usage", "help", "version", "pretend", "skipfile!", "skipfilename=s", - "processes=i", "tagseparator=s", "lameargs=s", "copyfiles" + "processes=i", "tagseparator=s", "lameargs=s", "dontcopy!", "delete" ); # info flag is the inverse of --quiet @@ -191,9 +200,13 @@ $cmdpath = which($cmd); } croak "$cmd not found" unless $cmdpath; - $Options{info} && msg("Using $cmd from: $cmdpath"); + msg_info("Using $cmd from: $cmdpath"); } +# Turn off unsyncing to improve compatibility with broken +# software (e.g. Logitech Media Server) +MP3::Tag->config(id3v23_unsync => 0); + # Convert directories to absolute paths $source_root = File::Spec->rel2abs($source_root); $target_root = File::Spec->rel2abs($target_root); @@ -201,50 +214,273 @@ die "Source directory not found: $source_root\n" unless -d $source_root; -# count all flac files in source_dir -# Display a progress report after each file, e.g. Processed 367/4394 files -# Possibly do some timing and add a Estimated Time Remaining -# Will need to only count files that are going to be processed. -# Hmmm could get complicated. - -$Options{info} - && msg( $pretendString . "Processing directory: $source_root" ); +msg_info( $pretendString . "Processing directory: $source_root" ); # Now look for files in the source dir # (following symlinks) +my %flac_mp3_files = get_all_paths('name', '.flac', $source_root, $target_root, '.mp3'); +my @flac_files = sort keys { %flac_mp3_files }; + +my $file_count = scalar @flac_files; +msg_info( "Found $file_count flac file" . ( $file_count > 1 ? 's' : '' . "\n" ) ); + +# If allowed, delete surplus files and folders from target directory, keeping +# it in perfect sync with (i.e. a mirror of) the source directory. +delete_excess_files_from_dest($source_root, $target_root) if ( $Options{delete} ) ; + +# Traverse all flac-files, but process only those who doesn't need transcoding. +# Save for later the name and size of each of the flac files which need transcoding. +my %flac_file_size = (); +my $total_file_size_to_transcode = 0; +my $cntr_processed = 0; +my $cntr_all; +my $t0 = 0; +foreach my $src_file (@flac_files) { + if ( $Options{force} || ( (my $result = path_and_conversion($src_file,0)) == 1 ) ) { + my $filesize = -s $src_file; + # save size data in a hash + $flac_file_size{$src_file} = $filesize; + $total_file_size_to_transcode += $filesize; + } + else { + $cntr_processed++; + if ( scalar @$result) { + msg_info(""); + foreach my $string ( @$result ) { + msg_info($string); + } + } + }; + $cntr_all ++; -my @flac_files = @{ find_files( $source_root, qr/\.flac$/i ) }; + # Show the progress every second + if ( $Options{info} && + ( (!$Options{force}) || ((time - $t0) >= 1) || ($cntr_all==$file_count) ) ) { + $t0 = time; + print("\r" . $cntr_processed . " of " . $cntr_all . " flac files were processed without transcoding."); + }; -# Get directories from target_dir and put in an array -my ( $target_root_volume, $target_root_path, $target_root_file ) = File::Spec->splitpath( $target_root, 1 ); -my @target_root_elements = File::Spec->splitdir($target_root_path); +} +msg_info(""); + + +my $files_to_trancode = scalar (keys %flac_file_size); +if ($files_to_trancode) { + my $t0 = time; # starting time for the transcoding part + my $files_transcoded_cntr = 0; + my $size_transcoded_so_far = 0; + + msg_info(""); + msg_info("The remaining $files_to_trancode files will be transcoded."); + msg_info("Total size of files to convert: " . + sprintf("%.1f", $total_file_size_to_transcode/(1024**2)) . " MB"); + # use parallel processing to launch multiple transcoding processes + msg_info("Using $Options{processes} transcoding processes.\n"); + my $pm = new Parallel::ForkManager($Options{processes}); + + $pm->run_on_finish( sub { + # This callback code is run after each transcode and outputs messages generated + # by the children, as well as overall progress info + # + # According to the Parallel::Forkmanager documentation, the structure of the + # input data @_ to this callback function is as follows: + # ($pid, $exit_code, $ident, $exit_signal, $core_dump, $data_structure_reference) + my $src = $_[2]; # this is "$ident" + my $messages = $_[5]; # this is "$data_structure_reference" + + # Update counter of transcoded files + $files_transcoded_cntr++; + + # display information about the transcoded/tagged file + foreach my $string ( @$messages ) { + msg_info($string); + } + # display updated progress info + msg_info("Processed " . + ($cntr_processed + $files_transcoded_cntr) . "/$file_count files."); + my $elapsedtime = time - $t0; + $size_transcoded_so_far += $flac_file_size{$src}; + + # After 20 seconds, we have enough data to provide a qualified guess + # about the estimated finish time. + if ($elapsedtime > 20) { + my $remains_to_be_transcoded + = $total_file_size_to_transcode - $size_transcoded_so_far; + my $transcoding_rate = $size_transcoded_so_far / $elapsedtime; + my $estim_finish_time + = localtime(time + $remains_to_be_transcoded / $transcoding_rate); + msg_info("Estimated finish time is $estim_finish_time."); + }; + + msg_info(""); + }); + + # Transcoding loop starts here + foreach my $src_file (sort keys %flac_file_size) { + $pm->start($src_file) and next; # forks here + + # transcode and generate messages with file info + my $messageref = path_and_conversion($src_file,1); + + # terminates the child process, send messages to callback sub + $pm->finish(0, $messageref ); + } + $pm->wait_all_children; -# use parallel processing to launch multiple transcoding processes -msg("Using $Options{processes} transcoding processes.\n"); -my $pm = new Parallel::ForkManager($Options{processes}); -foreach my $src_file (@flac_files) { - $pm->start and next; # Forks and returns the pid for the child - path_and_conversion($src_file); - $pm->finish; # Terminates the child process +}; + +# If allowed, copy non-flac files to destination dirs +copy_non_flacs($source_root, $target_root) unless ( $Options{dontcopy} ); + +1; +# ------------ Main program ends here -------------------------------------- + + +# ------------ Subroutines start here -------------------------------------- + +sub delete_excess_files_from_dest { + my ($source_root, $target_root) = @_; + + # Generate (source => target) hashes for the files found using + # each of the following combinations of root dirs and file suffixes + my %existing_target_mp3_files = get_all_paths('name', '.mp3', $target_root, $target_root, '.mp3'); + my %existing_source_mp3_files = get_all_paths('name', '.mp3', $source_root, $target_root, '.mp3'); + my %non_flac_files = get_all_paths('not_name', '.flac', $source_root, $target_root, ''); + my %existing_target_non_mp3_files = get_all_paths('not_name', '.mp3', $target_root, $target_root, ''); + + # 1. calculate what files to expect in directory after finished transcoding and copying + my @expected_transcoded_mp3s = keys { reverse %flac_mp3_files }; # expected mp3 files in target from transcoded flac files + my @expected_copied_files = keys { reverse %non_flac_files }; # expected files in target copied from non-flac files in source + my @expected_files = uniq(@expected_transcoded_mp3s, @expected_copied_files); # Join the arrays and remove any duplicates + + # 2. check what files are actually present + my @actual_mp3s = keys { reverse %existing_target_mp3_files }; # actual existing mp3 files in target + my @actual_non_mp3s = keys { reverse %existing_target_non_mp3_files }; # existing non-mp3 files in target + my @actual_files = (@actual_mp3s, @actual_non_mp3s); # Join the arrays (being mutually exclusive, there is no overlap) + + # 3. determine which files to remove from target directory tree + my @files_to_remove = single_difference(\@expected_files, \@actual_files); + + # 4. determine which subdirectories to remove from target directory tree + my @expected_subdirs_in_target = get_all_dirs($source_root,$target_root); + my @actual_subdirs_in_target = get_all_dirs($target_root,$target_root); + my @dirs_to_remove = single_difference(\@expected_subdirs_in_target, \@actual_subdirs_in_target); + + # 5. carry out the deletions + foreach my $file (@files_to_remove) { + $Options{pretend} || unlink $file or die "Unable to delete $file: $!"; + msg_info($pretendString . "Deleted \"$file\""); + } + foreach my $dir (reverse sort @dirs_to_remove) { + $Options{pretend} || File::Path->remove_tree($dir) or die "Unable to delete directory $dir: $!"; + msg_info($pretendString . "Deleted directory \"$dir\""); + } +} + +# Return all unique elements of input array @_ +sub uniq { + return sort keys %{{ map { $_ => 1 } @_ }} +}; + +# Acccept two arrays @A and @B as argument, return elements in @B that aren't in @A. +sub single_difference { + my ($A, $B) = @_; + + # build lookup table + my %seen = (); + my @bonly = (); + @seen{@$A} = (1) x @$A; + foreach my $item (@$B) { + push(@bonly, $item) unless $seen{$item}; + } + return sort @bonly; } -$pm->wait_all_children; +sub get_all_dirs { + my ($root, $new_root) = @_; + # we supply no suffix, so we search for directories (not files): + my @orig_dirs = @{ find_files_or_dirs($root) }; + + my @dirs = (); + foreach my $dir (@orig_dirs) { + # strip source root dir from path... + my $rel_path = File::Spec->abs2rel( $dir, $root ); + # then replace it with target root dir + push @dirs, File::Spec->rel2abs( $rel_path, $new_root ); + } + return sort @dirs; +} + +sub get_all_paths { + my ($rule, $suffix, $root, $new_root, $new_suffix) = @_; + my @orig_files = @{ find_files_or_dirs($root, $rule, $suffix) }; + + # Even if $root = $new_root, we need to do the following operations + # to get a consistent path format (otherwise problematic in e.g. MS Win) + # that is suitable for later string comparison: + my %paths = (); + foreach my $src (@orig_files) { + # Strip source root dir from file path + my $rel_path = File::Spec->abs2rel( $src, $root ); + # ... then replace it with target root dir and change file suffix. + ($paths{$src} = File::Spec->rel2abs( $rel_path, $new_root ) ) =~ s{$suffix$}{$new_suffix}xmsi; + } + return %paths +} -if ( $Options{copyfiles} ) { - my @non_flac_files - = sort File::Find::Rule->file()->extras( { follow => 1 } )->not_name(qr/\.flac$/i) - ->in($source_root); +sub find_files_or_dirs { + my $path = shift; + my $rule = shift; + my $suffix = shift; + + # If a matching rule and file suffix is defined we are looking for files, + # otherwise we are looking for directories. + my $found_list; + if (defined $rule && defined $suffix) { + $found_list = File::Find::Rule->file()->extras( { follow => 1 } )->$rule(qr{$suffix$}xmsi) + } + else { + $found_list = File::Find::Rule->directory->extras( { follow => 1 } ) + }; + + # skip any directories where a "skipfile" is found + my @found; + if ( $Options{skipfile} && ($path eq $source_root) ) { + my $skip_list = File::Find::Rule->directory->exec( + sub { + my ( $fname, $fpath, $frpath ) = @_; + if ( -f File::Spec->catdir( $frpath, $Options{skipfilename} ) ) { + return 1; + } + else { + return 0; + } + } + )->prune->discard; + @found = sort File::Find::Rule->or( $skip_list, $found_list )->in($path); + } + else { + @found = sort $found_list ->in($path); + } +return \@found; +} + +sub copy_non_flacs { + my ($source_root, $target_root) = @_; + + my %non_flac_files = get_all_paths('not_name', '.flac', $source_root, $target_root, ''); + my @non_flac_files = keys %non_flac_files; my $non_flac_file_count = scalar @non_flac_files; - $Options{info} && - msg( "Found $non_flac_file_count non-flac file" .( $non_flac_file_count != 1 ? 's' : '' . "\n" ) ); + msg_info( "Found $non_flac_file_count non-flac file" . + ( $non_flac_file_count != 1 ? 's' : '' . "\n" ) ); # Copy non-flac files from source to dest directories my $t0 = time; my $cntr_all = 0; my $cntr_copied = 0; foreach my $src_file (@non_flac_files) { - my ($dst_dir, $dst_file) = get_dest_file_path_non_flac($src_file); - # Flag which determines if file should be copied: + my $dst_file = $non_flac_files{$src_file}; + # Flag which determines if file should be copied: my $do_copy = 1; # Don't copy file if it already exists in dest directory and # has identical md5 to the source file @@ -256,11 +492,13 @@ }; } else { - # Create the destination directory if it - # doesn't already exist - mkpath($dst_dir) - or die "Can't create directory $dst_dir\n" - unless -d $dst_dir; + # Create the destination directory if it + # doesn't already exist + (undef, my $dst_dir) = + File::Basename::fileparse($dst_file); # retrieve directory name + unless ( $Options{pretend} || -d $dst_dir ) { + mkpath($dst_dir) or die "Can't create directory $dst_dir\n"; + } }; if ( $do_copy ) { unless ( $Options{pretend} ) { @@ -270,35 +508,14 @@ }; $cntr_all ++; # Show the progress every second - if ( ((time - $t0) >= 1) || ($cntr_all==$non_flac_file_count) ) { + if ( $Options{info} && + ( ((time - $t0) >= 1) || ($cntr_all==$non_flac_file_count) ) ) { $t0 = time; - print("\r" . $pretendString . $cntr_copied . " non-flac files of " . $cntr_all ." were copied to dest directories."); + print("\r" . $pretendString . $cntr_copied . + " non-flac files of " . $cntr_all ." were copied to dest directories."); }; }; - msg("\n"); # double line feed -}; - - -sub get_dest_file_path_non_flac { - my $source = shift; - - # remove $source_dir from front of $src_file - my $target = $source; - $target =~ s{\Q$source_root/\E}{}xms; - - # Get directories in target and put in an array - # Note: the filename is the source file name - my ( $target_volume, $target_path, $source_file ) = File::Spec->splitpath($target); - my @target_path_elements = File::Spec->splitdir($target_path); - - # Add the dst_dirs to the dst root and join back together - $target_path = File::Spec->catdir( @target_root_elements, @target_path_elements ); - - # Now join it all together to get the complete path of the dest_file - $target = File::Spec->catpath( $target_root_volume, $target_path, $source_file ); - my $target_dir = File::Spec->catpath( $target_root_volume, $target_path, '' ); - - return $target_dir,$target; + msg_info("\n"); # double line feed }; sub get_md5_of_non_flac_file { @@ -310,120 +527,95 @@ sub get_md5_of_non_flac_file { return $md5_code; }; -# use parallel processing to launch multiple transcoding processes sub path_and_conversion{ - my $source = shift; + # When "$transcode_enabled" is false it means that we are + # checking whether the file should be transcoded. If so, we + # return, and no further processing is taking place during this call. + # If the file is not to be transcoded, we stay and update tags if + # necessary. + + my $source = shift; + my $target = $flac_mp3_files{$source}; + my $transcode_enabled = shift; + my @messages = (); + my $t0; + my $elapsed_time = undef; + + $Options{debug} && msg("source: '$source'"); + $Options{debug} && msg("target: '$target'"); - # remove $source_dir from front of $src_file - my $target = $source; - $target =~ s{\Q$source_root/\E}{}xms; + # Step 1: get tags from flac file + my $source_tags = read_flac_tags($source); - # Get directories in target and put in an array - # Note: the filename is the source file name - my ( $target_volume, $target_path, $source_file ) = File::Spec->splitpath($target); - my @target_path_elements = File::Spec->splitdir($target_path); + # Step 2: hash to hold tags that will be updated + my $tags_to_update = preprocess_flac_tags( $source_tags ); - # Add the dst_dirs to the dst root and join back together - $target_path = File::Spec->catdir( @target_root_elements, @target_path_elements ); - # Add volume for OSes that require it (MSWin etc.) - $target_path = File::Spec->catpath( $target_root_volume, $target_path, '' ); + # Step 3: Initialise file processing flags + my ($pflags, $mess) = examine_destfile_tags( $target, $tags_to_update ); + push @messages, @$mess; - # Get the basename of the dst file - my ( $target_base, $target_dir, $source_ext ) = fileparse( $source_file, qr{\Q.flac\E$}xmsi ); + if ( ( !$$pflags{exists} || $$pflags{md5} || $Options{force} ) + && !$Options{tagsonly} ) { + + # Return if this file would be transcoded + return 1 unless ($transcode_enabled); + + $t0 = time; + # Step 4: Transcode the file based on the processing flags + $mess = transcode_file( $source, $target, $pflags ); + push @messages, @$mess; + $elapsed_time = time - $t0; + }; - # Now join it all together to get the complete path of the dest_file - $target = File::Spec->catpath( $target_volume, $target_path, $target_base . '.mp3' ); + # Step 5: Write the tags based on the processing flags + $mess = write_tags( $target, $tags_to_update, $pflags ); + push @messages, @$mess; - convert_file( $source, $target ); + if (defined $elapsed_time) { + push @messages, sprintf("Conversion took %5.2f seconds.", $elapsed_time); + }; + + return \@messages; }; -1; - -sub find_files { - my $path = shift; - my $regex = shift; - - my @found_files; - - my $found_list = File::Find::Rule->extras( { follow => 1 } )->name($regex); - if ( $Options{skipfile} ) { - my $skip_list = File::Find::Rule->directory->exec( - sub { - my ( $fname, $fpath, $frpath ) = @_; - if ( -f File::Spec->catdir( $frpath, $Options{skipfilename} ) ) { - return 1; - } - else { - return 0; - } - } - )->prune->discard; - @found_files = sort File::Find::Rule->or( $skip_list, $found_list )->in($path); - } - else { - - @found_files = sort $found_list ->in($path); - } - - $Options{debug} && msg( Dumper(@found_files) ); - - if ( $Options{info} ) { - my $file_count = scalar @found_files; - msg( "Found $file_count flac file" . ( $file_count > 1 ? 's' : '' . "\n" ) ); - } - - return \@found_files; -} - sub showusage { print <<"EOT"; -Usage: $0 [--pretend] [--quiet] [--debug] [--tagsonly] [--force] [--tagdiff] [--noskipfile] [--skipfilename=] [--lameargs='parameter-list'] --pretend Don't actually do anything + +Usage: $0 [--pretend] [--quiet] [--debug] [--tagsonly] [--force] [--tagdiff] [--noskipfile] [--skipfilename=] [--lameargs='parameter-list'] + + --pretend Don't actually do anything --quiet Disable informational output to stdout --debug Enable debugging output. For developers only! --tagsonly Don't do any transcoding - just update tags --force Force transcoding and tag update even if not required --tagdiff Print source/dest tag values if different --lameargs='s' specify parameter(string) to be passed to the LAME Encoder - Default: "--noreplaygain --vbr-new -V 2 -h --nohist --quiet" + Default: "--noreplaygain -V 2 -h --nohist --quiet" --noskipfile Ignore any skip files --skipfilename Specify the name of the skip file. Default: flac2mp3.ignore - --processes=n Launch n parallel transcoding processes (does not work on Windows platform) + --processes=n Launch n parallel transcoding processes (limited support on Windows platform) Use with multi-core CPUs. Default: 1 --tagseparator=s Use "s" as the separator to join multiple instances of the same tag. Default: "/" - --copyfiles Copy non-flac files to dest directories + --dontcopy Don't copy non-flac files to dest directories + --delete Delete surplus files and directories in destination, keeping in sync with source dir + +Example: $0 --processes=6 --delete ~/FLAC/ ~/MP3/ + EOT exit 0; } -sub msg { - my $msg = shift; - print "$msg\n"; +sub msg { + print "@_\n" } -sub convert_file { - my ( $source, $target ) = @_; - - $Options{debug} && msg("source: '$source'"); - $Options{debug} && msg("target: '$target'"); - - # get tags from flac file - my $source_tags = read_flac_tags($source); - - # hash to hold tags that will be updated - my $tags_to_update = preprocess_flac_tags( $source_tags ); - - # Initialise file processing flags - my $pflags = examine_destfile_tags( $target, $tags_to_update ); - - # Transcode the file based on the processing flags - transcode_file( $source, $target, $pflags ); - - # Write the tags based on the processing flags - write_tags( $target, $tags_to_update, $pflags ); +sub msg_info { + # display only if "--quiet" option is not in use + $Options{info} && msg(@_) } sub read_flac_tags { @@ -512,6 +704,7 @@ sub examine_destfile_tags { my $destfilename = shift; my $frames_ref = shift; my %frames_to_update = %$frames_ref; # this is only to minimize changes + my @return_messages = (); # Initialise file processing flags my %pflags = ( @@ -621,11 +814,12 @@ sub examine_destfile_tags { if ( $dest_text ne $srcframe ) { $pflags{tags} = 1; if ( $Options{tagdiff} ) { - msg("frame: '$frame'"); - msg("srcframe value: '$srcframe'"); - msg("destframe value: '$dest_text'"); + push @return_messages, ( + "frame: '$frame'", + "srcframe value: '$srcframe'", + "destframe value: '$dest_text'" + ); } - } } } @@ -647,7 +841,7 @@ sub examine_destfile_tags { msg( Dumper \%frames_to_update ); } - return \%pflags; + return \%pflags, \@return_messages; } sub transcode_file { @@ -655,93 +849,92 @@ sub transcode_file { my $target = shift; my $pflags_ref = shift; my %pflags = %$pflags_ref; # this is only to minimize changes + my @return_messages = (); + my $elapsed_time; + + # Transcode to a temp file in the destdir. + # Rename the file if the conversion completes sucessfully + # This avoids leaving incomplete files in the destdir + # If we're "pretending", don't create a File::Temp object + my $tmpfilename; + my $tmpfh; + if ( $Options{pretend} ) { + $tmpfilename = $target; + } + else { + # retrieve destination directory name + (undef, my $dst_dir) = File::Basename::fileparse($target); - my ( $target_volume, $target_dir, $target_filename ) = File::Spec->splitpath($target); - my $dst_dir = File::Spec->catpath( $target_volume, $target_dir, '' ); - - if ( ( !$pflags{exists} || $pflags{md5} || $Options{force} ) - && !$Options{tagsonly} ) - { - - # Transcode to a temp file in the destdir. - # Rename the file if the conversion completes sucessfully - # This avoids leaving incomplete files in the destdir - # If we're "pretending", don't create a File::Temp object - my $tmpfilename; - my $tmpfh; - if ( $Options{pretend} ) { - $tmpfilename = $target; - } - else { - - # Create the destination directory if it - # doesn't already exist - unless (-d $dst_dir) { - # If necessary, allow a second check. Don't die just because the - # dir was created by another child (race condition): - mkpath($dst_dir) or (-d $dst_dir) - or die "Can't create directory $dst_dir\n"; - }; - $tmpfh = new File::Temp( - UNLINK => 1, - DIR => $dst_dir, - SUFFIX => '.tmp' - ); - $tmpfilename = $tmpfh->filename; - } - $Options{info} - && msg( $pretendString . "Transcoding \"$source\"" ); - - my $convert_command = "\"$flaccmd\" @flacargs \"$source\"" . "| \"$lamecmd\" @lameargs - \"$tmpfilename\""; + # Create the destination directory if it + # doesn't already exist + unless (-d $dst_dir) { + # If necessary, allow a second check. Don't die just because the + # dir was created by another child (race condition): + mkpath($dst_dir) or (-d $dst_dir) + or die "Can't create directory $dst_dir\n"; + }; + $tmpfh = new File::Temp( + UNLINK => 1, + DIR => $dst_dir, + SUFFIX => '.tmp' + ); + $tmpfilename = $tmpfh->filename; + } + # Save message to be displayed on screen + push @return_messages, $pretendString . "Transcoding \"$source\"" ; + + my $convert_command = + "\"$flaccmd\" @flacargs \"$source\"" . "| \"$lamecmd\" @lameargs - \"$tmpfilename\""; - $Options{debug} && msg("transcode: $convert_command"); + $Options{debug} && msg("transcode: $convert_command"); - # Convert the file (unless we're pretending} - my $exit_value; - if ( !$Options{pretend} ) { - $exit_value = system($convert_command); - } - else { - $exit_value = 0; - } + # Convert the file (unless we're pretending} + my $exit_value; + if ( !$Options{pretend} ) { + $exit_value = system($convert_command); + } + else { + $exit_value = 0; + } - $Options{debug} - && msg("Exit value from convert command: $exit_value"); + $Options{debug} + && msg("Exit value from convert command: $exit_value"); - if ($exit_value) { - msg("$convert_command failed with exit code $exit_value"); + if ($exit_value) { + msg("$convert_command failed with exit code $exit_value"); - # delete the destfile if it exists - unlink $tmpfilename; + # delete the destfile if it exists + unlink $tmpfilename; - # should check exit status of this command + # should check exit status of this command + exit($exit_value); + } - exit($exit_value); - } + if ( !$Options{pretend} ) { - if ( !$Options{pretend} ) { + # If we get here, assume the conversion has succeeded + $tmpfh->unlink_on_destroy(0); + $tmpfh->close; + croak "Failed to rename '$tmpfilename' to '$target' $!" + unless rename( $tmpfilename, $target ); - # If we get here, assume the conversion has succeeded - $tmpfh->unlink_on_destroy(0); - $tmpfh->close; - croak "Failed to rename '$tmpfilename' to '$target' $!" - unless rename( $tmpfilename, $target ); + # the destfile now exists! + $pflags{exists} = 1; - # the destfile now exists! - $pflags{exists} = 1; + # and the tags need writing + $pflags{tags} = 1; + } - # and the tags need writing - $pflags{tags} = 1; - } - } if ( $Options{debug} ) { msg("pf_exists: $pflags{exists}"); msg("pf_tags: $pflags{tags}"); msg( "\$Options{pretend}: " . ( $Options{pretend} ? 'set' : 'not set' ) ); - } - + } + %$pflags_ref = %pflags; # this is only to minimize changes + + return \@return_messages; } sub write_tags { @@ -750,16 +943,13 @@ sub write_tags { my $pflags_ref = shift; my %frames_to_update = %$frames_ref; # this is only to minimize changes my %pflags = %$pflags_ref; # this is only to minimize changes + my @return_messages = (); # Write the tags - if ($pflags{exists} - && ( $pflags{tags} - || $Options{force} ) - ) - { + if ( $pflags{exists} && ( $pflags{tags} || $Options{force} ) ) { - $Options{info} - && msg( $pretendString . "Writing tags to \"$destfilename\"" ); + # save message to be displayed on screen + push @return_messages, $pretendString . "Writing tags to \"$destfilename\""; if ( !$Options{pretend} ) { my $mp3 = MP3::Tag->new($destfilename); @@ -822,6 +1012,7 @@ sub write_tags { # utime $srcstat->mtime, $srcstat->mtime, $destfilename; } } + return \@return_messages; } sub INT_Handler { @@ -847,8 +1038,7 @@ sub fixUpTrackNumber { $trackNum = sprintf( "%02u", $trackNum ); } else { - $Options{info} - && msg('TRACKNUMBER not numeric'); + msg_info('TRACKNUMBER not numeric'); } } return $trackNum; @@ -898,4 +1088,4 @@ sub picsToAPICframes { # vim:set softtabstop=4: # vim:set shiftwidth=4: -__END__ +__END__ \ No newline at end of file diff --git a/lib/MP3/Tag/ID3v2.pm b/lib/MP3/Tag/ID3v2.pm index 273d625..e7e3934 100755 --- a/lib/MP3/Tag/ID3v2.pm +++ b/lib/MP3/Tag/ID3v2.pm @@ -636,6 +636,12 @@ sub insert_space { ); my $tempfile = $tempfh->filename; + # Change file permissions to default value. + # Permissions will otherwise (at least on *nix) + # be 0600 regardless of umask, due a peculiar choice choice + # by the developers of the File::Temp module. + chmod umask ^ 0666, $tempfile; + if ($@) { warn "Can't open '$tempfile' to insert tag\n"; return -1; diff --git a/lib/Parallel/Forkmanager.pm b/lib/Parallel/Forkmanager.pm index c3d9491..29471a8 100644 --- a/lib/Parallel/Forkmanager.pm +++ b/lib/Parallel/Forkmanager.pm @@ -6,7 +6,7 @@ Parallel::ForkManager - A simple parallel processing fork manager use Parallel::ForkManager; - $pm = new Parallel::ForkManager($MAX_PROCESSES); + $pm = Parallel::ForkManager->new($MAX_PROCESSES); foreach $data (@all_data) { # Forks and returns the pid for the child: @@ -39,7 +39,7 @@ The code for a downloader would look something like this: ... # Max 30 processes for parallel download - my $pm = new Parallel::ForkManager(30); + my $pm = Parallel::ForkManager->new(30); foreach my $linkarray (@links) { $pm->start and next; # do the fork @@ -83,11 +83,11 @@ will be forked. This is intended for debugging purposes. The optional second parameter, $tempdir, is only used if you want the children to send back a reference to some data (see RETRIEVING DATASTRUCTURES -below). If not provided, it is set to $L->tmpdir(). +below). If not provided, it is set to $L->tmpdir() is used. +$Lnew(10); for my $link (@ARGV) { $pm->start and next; my ($fn)= $link =~ /^.*\/(.*?)$/; @@ -236,25 +236,22 @@ Example of a program using callbacks to get child exit codes: my @names = qw( Fred Jim Lily Steve Jessica Bob Dave Christine Rico Sara ); # hash to resolve PID's back to child specific information - my $pm = new Parallel::ForkManager($max_procs); + my $pm = Parallel::ForkManager->new($max_procs); # Setup a callback for when a child finishes up so we can # get it's exit code - $pm->run_on_finish( - sub { my ($pid, $exit_code, $ident) = @_; + $pm->run_on_finish( sub { + my ($pid, $exit_code, $ident) = @_; print "** $ident just got out of the pool ". "with PID $pid and exit code: $exit_code\n"; - } - ); + }); - $pm->run_on_start( - sub { my ($pid,$ident)=@_; + $pm->run_on_start( sub { + my ($pid,$ident)=@_; print "** $ident started, pid: $pid\n"; - } - ); + }); - $pm->run_on_wait( - sub { + $pm->run_on_wait( sub { print "** Have to wait for one children ...\n" }, 0.5 @@ -282,7 +279,7 @@ In this simple example, each child sends back a string reference. use Parallel::ForkManager 0.7.6; use strict; - my $pm = new Parallel::ForkManager(2, '/server/path/to/temp/dir/'); + my $pm = Parallel::ForkManager->new(2, '/server/path/to/temp/dir/'); # data structure retrieval and handling $pm -> run_on_finish ( # called BEFORE the first call to start() @@ -326,7 +323,7 @@ process whatever is retrieved. use Data::Dumper; # to display the data structures retrieved. use strict; - my $pm = new Parallel::ForkManager(20); # using the system temp dir $L->tmpdir() + my $pm = Parallel::ForkManager->new(20); # using the system temp dir $L $processes, processes => {}, @@ -452,14 +453,17 @@ sub new { my ($c,$processes, $tempdir)=@_; }; # determine temporary directory for storing data structures - $tempdir = File::Spec->tmpdir() unless (defined($tempdir) && length($tempdir)); - die qq|Temporary directory "$tempdir" doesn't exist or is not a directory.| unless (-e $tempdir && -d _); # ensure temp dir exists and is indeed a directory - $h->{tempdir} = $tempdir; # add tempdir to Parallel::ForkManager object so children can use it + # add it to Parallel::ForkManager object so children can use it + # We don't let it clean up so it won't do it in the child process + # but we have our own DESTROY to do that. + $h->{tempdir} = File::Temp::tempdir(CLEANUP => 0); return bless($h,ref($c)||$c); }; -sub start { my ($s,$identification)=@_; +sub start { + my ($s,$identification)=@_; + die "Cannot start another process while you are in the child process" if $s->{in_child}; while ($s->{max_proc} && ( keys %{ $s->{processes} } ) >= $s->{max_proc}) { @@ -484,7 +488,9 @@ sub start { my ($s,$identification)=@_; } } -sub finish { my ($s, $x, $r)=@_; +sub finish { + my ($s, $x, $r)=@_; + if ( $s->{in_child} ) { if (defined($r)) { # store the child's data structure my $storable_tempfile = File::Spec->catfile($s->{tempdir}, 'Parallel-ForkManager-' . $s->{parent_pid} . '-' . $$ . '.txt'); @@ -504,7 +510,9 @@ sub finish { my ($s, $x, $r)=@_; return 0; } -sub wait_children { my ($s)=@_; +sub wait_children { + my ($s)=@_; + return if !keys %{$s->{processes}}; my $kid; do { @@ -514,7 +522,9 @@ sub wait_children { my ($s)=@_; *wait_childs=*wait_children; # compatibility -sub wait_one_child { my ($s,$par)=@_; +sub wait_one_child { + my ($s,$par)=@_; + my $kid; while (1) { $kid = $s->_waitpid(-1,$par||=0); @@ -543,7 +553,9 @@ sub wait_one_child { my ($s,$par)=@_; $kid; }; -sub wait_all_children { my ($s)=@_; +sub wait_all_children { + my ($s)=@_; + while (keys %{ $s->{processes} }) { $s->on_wait; $s->wait_one_child(defined $s->{on_wait_period} ? &WNOHANG : undef); @@ -552,21 +564,29 @@ sub wait_all_children { my ($s)=@_; *wait_all_childs=*wait_all_children; # compatibility; -sub run_on_finish { my ($s,$code,$pid)=@_; +sub run_on_finish { + my ($s,$code,$pid)=@_; + $s->{on_finish}->{$pid || 0}=$code; } -sub on_finish { my ($s,$pid,@par)=@_; +sub on_finish { + my ($s,$pid,@par)=@_; + my $code=$s->{on_finish}->{$pid} || $s->{on_finish}->{0} or return 0; $code->($pid,@par); }; -sub run_on_wait { my ($s,$code, $period)=@_; +sub run_on_wait { + my ($s,$code, $period)=@_; + $s->{on_wait}=$code; $s->{on_wait_period} = $period; } -sub on_wait { my ($s)=@_; +sub on_wait { + my ($s)=@_; + if(ref($s->{on_wait}) eq 'CODE') { $s->{on_wait}->(); if (defined $s->{on_wait_period}) { @@ -576,15 +596,21 @@ sub on_wait { my ($s)=@_; }; }; -sub run_on_start { my ($s,$code)=@_; +sub run_on_start { + my ($s,$code)=@_; + $s->{on_start}=$code; } -sub on_start { my ($s,@par)=@_; +sub on_start { + my ($s,@par)=@_; + $s->{on_start}->(@par) if ref($s->{on_start}) eq 'CODE'; }; -sub set_max_procs { my ($s, $mp)=@_; +sub set_max_procs { + my ($s, $mp)=@_; + $s->{max_proc} = $mp; } @@ -596,7 +622,9 @@ sub _waitpid { # Call waitpid() in the standard Unix fashion. # On ActiveState Perl 5.6/Win32 build 625, waitpid(-1, &WNOHANG) always # blocks unless an actual PID other than -1 is given. -sub _NT_waitpid { my ($s, $pid, $par) = @_; +sub _NT_waitpid { + my ($s, $pid, $par) = @_; + if ($par == &WNOHANG) { # Need to nonblock on each of our PIDs in the pool. my @pids = keys %{ $s->{processes} }; # Simulate -1 (no processes awaiting cleanup.) @@ -620,4 +648,12 @@ sub _NT_waitpid { my ($s, $pid, $par) = @_; } } +sub DESTROY { + my ($self) = @_; + + if ($self->{parent_pid} == $$ && -d $self->{tempdir}) { + File::Path::remove_tree($self->{tempdir}); + } +} + 1; From 1b0cd729a481c2ff42882f2092f0c0711a97f83d Mon Sep 17 00:00:00 2001 From: Carl Asplund Date: Thu, 28 Feb 2013 00:45:27 +0100 Subject: [PATCH 03/15] Change option from "--dontcopy" to "--copyfiles" Non-flac files ar no longer copied by default. --- flac2mp3.pl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flac2mp3.pl b/flac2mp3.pl index 4f77697..4fb6fec 100755 --- a/flac2mp3.pl +++ b/flac2mp3.pl @@ -162,7 +162,7 @@ GetOptions( \%Options, "quiet!", "tagdiff", "debug!", "tagsonly!", "force!", "usage", "help", "version", "pretend", "skipfile!", "skipfilename=s", - "processes=i", "tagseparator=s", "lameargs=s", "dontcopy!", "delete" + "processes=i", "tagseparator=s", "lameargs=s", "copyfiles", "delete" ); # info flag is the inverse of --quiet @@ -330,7 +330,7 @@ }; # If allowed, copy non-flac files to destination dirs -copy_non_flacs($source_root, $target_root) unless ( $Options{dontcopy} ); +copy_non_flacs($source_root, $target_root) if ( $Options{copyfiles} ); 1; # ------------ Main program ends here -------------------------------------- @@ -600,7 +600,7 @@ sub showusage { --tagseparator=s Use "s" as the separator to join multiple instances of the same tag. Default: "/" - --dontcopy Don't copy non-flac files to dest directories + --copyfiles Copy non-flac files to dest directories --delete Delete surplus files and directories in destination, keeping in sync with source dir Example: $0 --processes=6 --delete ~/FLAC/ ~/MP3/ From 7847c06ea107ceaf8461fc79b0f504c5c9175aba Mon Sep 17 00:00:00 2001 From: Robin Bowes Date: Thu, 28 Feb 2013 00:49:00 +0000 Subject: [PATCH 04/15] Fix up usage text --- flac2mp3.pl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flac2mp3.pl b/flac2mp3.pl index 0d944b8..da4099c 100755 --- a/flac2mp3.pl +++ b/flac2mp3.pl @@ -377,7 +377,9 @@ sub find_files { sub showusage { print <<"EOT"; -Usage: $0 [--pretend] [--quiet] [--debug] [--tagsonly] [--force] [--tagdiff] [--noskipfile] [--skipfilename=] [--lameargs='parameter-list'] --pretend Don't actually do anything +Usage: $0 [--pretend] [--quiet] [--debug] [--tagsonly] [--force] [--tagdiff] [--noskipfile] [--skipfilename=] [--lameargs='parameter-list'] + + --pretend Don't actually do anything --quiet Disable informational output to stdout --debug Enable debugging output. For developers only! --tagsonly Don't do any transcoding - just update tags From 7d6ef76060e4fe9f7bc95310cd6214a819d20b91 Mon Sep 17 00:00:00 2001 From: Carl Asplund Date: Sat, 9 Mar 2013 23:50:21 +0100 Subject: [PATCH 05/15] Resetting to match Robin's develop branch Preparing for a a new fresh start with git flow, therefore resetting 'master' to match Robins 'develop' branch as of 2013-03-09 --- changelog.txt | 61 ---- flac2mp3.pl | 646 +++++++++++++++---------------------------- lib/MP3/Tag/ID3v2.pm | 3 +- 3 files changed, 230 insertions(+), 480 deletions(-) diff --git a/changelog.txt b/changelog.txt index 55bf1dc..d7fc23d 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,64 +1,3 @@ -Modified version by Carl Asplund, rev 5 (February 2013) - - Reintroduce progress reporting, this time with full support for parallel processing. The estimated - finish time is given after 20 s of transcoding, based on transcoding rate and the total file size - left to transcode. - - No longer output messages directly during transcoding or tagging of a file, but save them in a - buffer and use the callback function "run_on_finish" of Parallel::Forkmanager to display them when - the corresponding child exits. - - First perform a separate processing loop for files that don't require transcoding, while at the same time saving - the names of the files that do need transcoding. This first loop doesn't employ parallel processing, whereas the second, - transcoding loop does, through the use of the Parallel::Forkmanager module. - - Change file permissions of the tempfile/mp3-file in Tag::MP3::ID3v2 to default value (as determined by umask). - File permissions will otherwise (on *nix) be 0600 due to the odd ways of the File::Temp module (not observing the umask). - 0600 file permissions make the files invisible to some software (as is the case with Logitech Media Server). - - Turn off id3 v2.3 unsyncing - again - for increased compatibility with broken software. This time due - to inability of Logitech Media Server to cope with it. - - Update Parallel::Forkmanager to v1.0.2 - -Modified version by Carl Asplund, rev 4 (February 2013) - - Added option "--delete". When enabled, any files in the destination - directories which are not accounted for in the source directories will - be deleted before transcoding begins. In this way, the destination is kept perfectly in sync with the source, - even when files and/or directories have been renamed. - - Copying non-flac files now honors skipfiles (i.e. not copying directories with a skipfile) - - Refactored code for the translation of paths from source to target directories. - - Bugfix: Now properly suppresses all output with option "--quiet" - -Modified version by Carl Asplund, rev 3 (January 2013) - - First version which combines embedded album art, multi-CPU processing, and copying of - non-flac files present in the flac folders (which, for some, is another way of handling album art) - - Extensive refactoring of code, everything built up from latest revision in Robin - Bowes' repository: r157 - - Fixed problem with paths under MS Win systems: volume (eg. "C:") was not preserved in r157 - - Various small fixes of broken code in r157 (addressing e.g. tickets #94 and #134) - - Code for embedded album art now integrated into Robin's code for the other tags - - Using Parallel::Forkmanager instead of Proc::ParallelLoop which solves two problems: - 1. Bug in Proc::ParallelLoop casues crash for asymmeteric loads, module seems to be abandoned since 2003 - 2. Parallel::Forkmanager works under MS Win (Proc::ParallelLoop doesn't) - - Fixed race condition bug for creation of target directory (previous code broken, would sometimes skip transcoding files) - - Fixed race condition bug in MP3::Tag::ID3v2, which has severely broken temp file generation code - in original version (see https://rt.cpan.org/Public/Bug/Display.html?id=66768) - Thanks to Jason Rhinelander for posting a very useful patch on CPAN. - - Progress report and estimated-time-to-finish removed for now. Will adress this later (parallel - processing calls for another solution) - - Known problems: Parallel processing under MS Win is not perfectly stable, MP3::Tag will sometimes be unable to - rename temp files during heavy I/O operation (multiple parallel threads with lame and/or flac running), thus - skipping the file in question. However, parallel code works flawlessly on MS Win when actual transcoding - is replaced by sleep() or counting loops. Sometimes a short, 1 sec, sleep will suffice if unable to rename, - before successfully retrying rename operation. Problem seems to be unrelated to filename conflicts and - possibly goes deeper than the CPAN modules used. - -Modified version by Carl Asplund, rev 2 (November 2012) - - Non-flac files are copied to dest directory only when the source and target - md5 numbers don't match - - Progress is displayed during copying of non-flac files - - Added option "--dontcopy" (Don't copy non-flac files to dest directories) - -Modified version by Carl Asplund (August 2009) - - Embedded picture data (album art) in flac file is copied over to the mp3 file - - Estimated time to finish and progress is displayed during transcoding - - Non-flac files in the folders (jpg, mp3 files etc.) are copied over to destination folders - v0.3.0 - Remove File::Glob (it's in core) and add Text::Glob - Re-work & refactor the search/checking code diff --git a/flac2mp3.pl b/flac2mp3.pl index 4fb6fec..da4099c 100755 --- a/flac2mp3.pl +++ b/flac2mp3.pl @@ -33,7 +33,6 @@ use Scalar::Util qw/ looks_like_number /; use FreezeThaw qw/ cmpStr /; use Digest::MD5; -use Time::HiRes qw ( time ); # ------- User-config options start here -------- # Assume flac and lame programs are in the path. @@ -48,21 +47,13 @@ # c:/windows/system32/flac.exe # or # c:\\windows\\system32\\flac.exe - -my ($flaccmd, $lamecmd); -if ($^O eq 'MSWin32' or $^O eq 'MSWin64') { - # This is an example of typical Windows filepaths: - $flaccmd = q|C:\Program Files (x86)\FLAC\flac.exe|; - $lamecmd = q|C:\Program Files\LAME\lame.exe|; -} -else { - $flaccmd = 'flac'; - $lamecmd = 'lame'; -} +my $flaccmd = 'flac'; +my $lamecmd = 'lame'; # Modify lame options if required my @lameargs = qw ( --noreplaygain + --vbr-new -V 2 -h --nohist @@ -162,7 +153,7 @@ GetOptions( \%Options, "quiet!", "tagdiff", "debug!", "tagsonly!", "force!", "usage", "help", "version", "pretend", "skipfile!", "skipfilename=s", - "processes=i", "tagseparator=s", "lameargs=s", "copyfiles", "delete" + "processes=i", "tagseparator=s", "lameargs=s", "copyfiles" ); # info flag is the inverse of --quiet @@ -200,13 +191,9 @@ $cmdpath = which($cmd); } croak "$cmd not found" unless $cmdpath; - msg_info("Using $cmd from: $cmdpath"); + $Options{info} && msg("Using $cmd from: $cmdpath"); } -# Turn off unsyncing to improve compatibility with broken -# software (e.g. Logitech Media Server) -MP3::Tag->config(id3v23_unsync => 0); - # Convert directories to absolute paths $source_root = File::Spec->rel2abs($source_root); $target_root = File::Spec->rel2abs($target_root); @@ -214,273 +201,50 @@ die "Source directory not found: $source_root\n" unless -d $source_root; -msg_info( $pretendString . "Processing directory: $source_root" ); +# count all flac files in source_dir +# Display a progress report after each file, e.g. Processed 367/4394 files +# Possibly do some timing and add a Estimated Time Remaining +# Will need to only count files that are going to be processed. +# Hmmm could get complicated. + +$Options{info} + && msg( $pretendString . "Processing directory: $source_root" ); # Now look for files in the source dir # (following symlinks) -my %flac_mp3_files = get_all_paths('name', '.flac', $source_root, $target_root, '.mp3'); -my @flac_files = sort keys { %flac_mp3_files }; - -my $file_count = scalar @flac_files; -msg_info( "Found $file_count flac file" . ( $file_count > 1 ? 's' : '' . "\n" ) ); - -# If allowed, delete surplus files and folders from target directory, keeping -# it in perfect sync with (i.e. a mirror of) the source directory. -delete_excess_files_from_dest($source_root, $target_root) if ( $Options{delete} ) ; - -# Traverse all flac-files, but process only those who doesn't need transcoding. -# Save for later the name and size of each of the flac files which need transcoding. -my %flac_file_size = (); -my $total_file_size_to_transcode = 0; -my $cntr_processed = 0; -my $cntr_all; -my $t0 = 0; -foreach my $src_file (@flac_files) { - if ( $Options{force} || ( (my $result = path_and_conversion($src_file,0)) == 1 ) ) { - my $filesize = -s $src_file; - # save size data in a hash - $flac_file_size{$src_file} = $filesize; - $total_file_size_to_transcode += $filesize; - } - else { - $cntr_processed++; - if ( scalar @$result) { - msg_info(""); - foreach my $string ( @$result ) { - msg_info($string); - } - } - }; - $cntr_all ++; - # Show the progress every second - if ( $Options{info} && - ( (!$Options{force}) || ((time - $t0) >= 1) || ($cntr_all==$file_count) ) ) { - $t0 = time; - print("\r" . $cntr_processed . " of " . $cntr_all . " flac files were processed without transcoding."); - }; +my @flac_files = @{ find_files( $source_root, qr/\.flac$/i ) }; -} -msg_info(""); - - -my $files_to_trancode = scalar (keys %flac_file_size); -if ($files_to_trancode) { - my $t0 = time; # starting time for the transcoding part - my $files_transcoded_cntr = 0; - my $size_transcoded_so_far = 0; - - msg_info(""); - msg_info("The remaining $files_to_trancode files will be transcoded."); - msg_info("Total size of files to convert: " . - sprintf("%.1f", $total_file_size_to_transcode/(1024**2)) . " MB"); - # use parallel processing to launch multiple transcoding processes - msg_info("Using $Options{processes} transcoding processes.\n"); - my $pm = new Parallel::ForkManager($Options{processes}); - - $pm->run_on_finish( sub { - # This callback code is run after each transcode and outputs messages generated - # by the children, as well as overall progress info - # - # According to the Parallel::Forkmanager documentation, the structure of the - # input data @_ to this callback function is as follows: - # ($pid, $exit_code, $ident, $exit_signal, $core_dump, $data_structure_reference) - my $src = $_[2]; # this is "$ident" - my $messages = $_[5]; # this is "$data_structure_reference" - - # Update counter of transcoded files - $files_transcoded_cntr++; - - # display information about the transcoded/tagged file - foreach my $string ( @$messages ) { - msg_info($string); - } - # display updated progress info - msg_info("Processed " . - ($cntr_processed + $files_transcoded_cntr) . "/$file_count files."); - my $elapsedtime = time - $t0; - $size_transcoded_so_far += $flac_file_size{$src}; - - # After 20 seconds, we have enough data to provide a qualified guess - # about the estimated finish time. - if ($elapsedtime > 20) { - my $remains_to_be_transcoded - = $total_file_size_to_transcode - $size_transcoded_so_far; - my $transcoding_rate = $size_transcoded_so_far / $elapsedtime; - my $estim_finish_time - = localtime(time + $remains_to_be_transcoded / $transcoding_rate); - msg_info("Estimated finish time is $estim_finish_time."); - }; - - msg_info(""); - }); - - # Transcoding loop starts here - foreach my $src_file (sort keys %flac_file_size) { - $pm->start($src_file) and next; # forks here - - # transcode and generate messages with file info - my $messageref = path_and_conversion($src_file,1); - - # terminates the child process, send messages to callback sub - $pm->finish(0, $messageref ); - } - $pm->wait_all_children; +# Get directories from target_dir and put in an array +my ( $target_root_volume, $target_root_path, $target_root_file ) = File::Spec->splitpath( $target_root, 1 ); +my @target_root_elements = File::Spec->splitdir($target_root_path); -}; - -# If allowed, copy non-flac files to destination dirs -copy_non_flacs($source_root, $target_root) if ( $Options{copyfiles} ); - -1; -# ------------ Main program ends here -------------------------------------- - - -# ------------ Subroutines start here -------------------------------------- - -sub delete_excess_files_from_dest { - my ($source_root, $target_root) = @_; - - # Generate (source => target) hashes for the files found using - # each of the following combinations of root dirs and file suffixes - my %existing_target_mp3_files = get_all_paths('name', '.mp3', $target_root, $target_root, '.mp3'); - my %existing_source_mp3_files = get_all_paths('name', '.mp3', $source_root, $target_root, '.mp3'); - my %non_flac_files = get_all_paths('not_name', '.flac', $source_root, $target_root, ''); - my %existing_target_non_mp3_files = get_all_paths('not_name', '.mp3', $target_root, $target_root, ''); - - # 1. calculate what files to expect in directory after finished transcoding and copying - my @expected_transcoded_mp3s = keys { reverse %flac_mp3_files }; # expected mp3 files in target from transcoded flac files - my @expected_copied_files = keys { reverse %non_flac_files }; # expected files in target copied from non-flac files in source - my @expected_files = uniq(@expected_transcoded_mp3s, @expected_copied_files); # Join the arrays and remove any duplicates - - # 2. check what files are actually present - my @actual_mp3s = keys { reverse %existing_target_mp3_files }; # actual existing mp3 files in target - my @actual_non_mp3s = keys { reverse %existing_target_non_mp3_files }; # existing non-mp3 files in target - my @actual_files = (@actual_mp3s, @actual_non_mp3s); # Join the arrays (being mutually exclusive, there is no overlap) - - # 3. determine which files to remove from target directory tree - my @files_to_remove = single_difference(\@expected_files, \@actual_files); - - # 4. determine which subdirectories to remove from target directory tree - my @expected_subdirs_in_target = get_all_dirs($source_root,$target_root); - my @actual_subdirs_in_target = get_all_dirs($target_root,$target_root); - my @dirs_to_remove = single_difference(\@expected_subdirs_in_target, \@actual_subdirs_in_target); - - # 5. carry out the deletions - foreach my $file (@files_to_remove) { - $Options{pretend} || unlink $file or die "Unable to delete $file: $!"; - msg_info($pretendString . "Deleted \"$file\""); - } - foreach my $dir (reverse sort @dirs_to_remove) { - $Options{pretend} || File::Path->remove_tree($dir) or die "Unable to delete directory $dir: $!"; - msg_info($pretendString . "Deleted directory \"$dir\""); - } -} - -# Return all unique elements of input array @_ -sub uniq { - return sort keys %{{ map { $_ => 1 } @_ }} -}; - -# Acccept two arrays @A and @B as argument, return elements in @B that aren't in @A. -sub single_difference { - my ($A, $B) = @_; - - # build lookup table - my %seen = (); - my @bonly = (); - @seen{@$A} = (1) x @$A; - foreach my $item (@$B) { - push(@bonly, $item) unless $seen{$item}; - } - return sort @bonly; -} - -sub get_all_dirs { - my ($root, $new_root) = @_; - # we supply no suffix, so we search for directories (not files): - my @orig_dirs = @{ find_files_or_dirs($root) }; - - my @dirs = (); - foreach my $dir (@orig_dirs) { - # strip source root dir from path... - my $rel_path = File::Spec->abs2rel( $dir, $root ); - # then replace it with target root dir - push @dirs, File::Spec->rel2abs( $rel_path, $new_root ); - } - return sort @dirs; -} - -sub get_all_paths { - my ($rule, $suffix, $root, $new_root, $new_suffix) = @_; - my @orig_files = @{ find_files_or_dirs($root, $rule, $suffix) }; - - # Even if $root = $new_root, we need to do the following operations - # to get a consistent path format (otherwise problematic in e.g. MS Win) - # that is suitable for later string comparison: - my %paths = (); - foreach my $src (@orig_files) { - # Strip source root dir from file path - my $rel_path = File::Spec->abs2rel( $src, $root ); - # ... then replace it with target root dir and change file suffix. - ($paths{$src} = File::Spec->rel2abs( $rel_path, $new_root ) ) =~ s{$suffix$}{$new_suffix}xmsi; - } - return %paths +# use parallel processing to launch multiple transcoding processes +msg("Using $Options{processes} transcoding processes.\n"); +my $pm = new Parallel::ForkManager($Options{processes}); +foreach my $src_file (@flac_files) { + $pm->start and next; # Forks and returns the pid for the child + path_and_conversion($src_file); + $pm->finish; # Terminates the child process } +$pm->wait_all_children; -sub find_files_or_dirs { - my $path = shift; - my $rule = shift; - my $suffix = shift; - - # If a matching rule and file suffix is defined we are looking for files, - # otherwise we are looking for directories. - my $found_list; - if (defined $rule && defined $suffix) { - $found_list = File::Find::Rule->file()->extras( { follow => 1 } )->$rule(qr{$suffix$}xmsi) - } - else { - $found_list = File::Find::Rule->directory->extras( { follow => 1 } ) - }; - - # skip any directories where a "skipfile" is found - my @found; - if ( $Options{skipfile} && ($path eq $source_root) ) { - my $skip_list = File::Find::Rule->directory->exec( - sub { - my ( $fname, $fpath, $frpath ) = @_; - if ( -f File::Spec->catdir( $frpath, $Options{skipfilename} ) ) { - return 1; - } - else { - return 0; - } - } - )->prune->discard; - @found = sort File::Find::Rule->or( $skip_list, $found_list )->in($path); - } - else { - @found = sort $found_list ->in($path); - } -return \@found; -} -sub copy_non_flacs { - my ($source_root, $target_root) = @_; - - my %non_flac_files = get_all_paths('not_name', '.flac', $source_root, $target_root, ''); - my @non_flac_files = keys %non_flac_files; +if ( $Options{copyfiles} ) { + my @non_flac_files + = sort File::Find::Rule->file()->extras( { follow => 1 } )->not_name(qr/\.flac$/i) + ->in($source_root); my $non_flac_file_count = scalar @non_flac_files; - msg_info( "Found $non_flac_file_count non-flac file" . - ( $non_flac_file_count != 1 ? 's' : '' . "\n" ) ); + $Options{info} && + msg( "Found $non_flac_file_count non-flac file" .( $non_flac_file_count != 1 ? 's' : '' . "\n" ) ); # Copy non-flac files from source to dest directories my $t0 = time; my $cntr_all = 0; my $cntr_copied = 0; foreach my $src_file (@non_flac_files) { - my $dst_file = $non_flac_files{$src_file}; - # Flag which determines if file should be copied: + my ($dst_dir, $dst_file) = get_dest_file_path_non_flac($src_file); + # Flag which determines if file should be copied: my $do_copy = 1; # Don't copy file if it already exists in dest directory and # has identical md5 to the source file @@ -492,13 +256,11 @@ sub copy_non_flacs { }; } else { - # Create the destination directory if it - # doesn't already exist - (undef, my $dst_dir) = - File::Basename::fileparse($dst_file); # retrieve directory name - unless ( $Options{pretend} || -d $dst_dir ) { - mkpath($dst_dir) or die "Can't create directory $dst_dir\n"; - } + # Create the destination directory if it + # doesn't already exist + mkpath($dst_dir) + or die "Can't create directory $dst_dir\n" + unless -d $dst_dir; }; if ( $do_copy ) { unless ( $Options{pretend} ) { @@ -508,14 +270,35 @@ sub copy_non_flacs { }; $cntr_all ++; # Show the progress every second - if ( $Options{info} && - ( ((time - $t0) >= 1) || ($cntr_all==$non_flac_file_count) ) ) { + if ( ((time - $t0) >= 1) || ($cntr_all==$non_flac_file_count) ) { $t0 = time; - print("\r" . $pretendString . $cntr_copied . - " non-flac files of " . $cntr_all ." were copied to dest directories."); + print("\r" . $pretendString . $cntr_copied . " non-flac files of " . $cntr_all ." were copied to dest directories."); }; }; - msg_info("\n"); # double line feed + msg("\n"); # double line feed +}; + + +sub get_dest_file_path_non_flac { + my $source = shift; + + # remove $source_dir from front of $src_file + my $target = $source; + $target =~ s{\Q$source_root/\E}{}xms; + + # Get directories in target and put in an array + # Note: the filename is the source file name + my ( $target_volume, $target_path, $source_file ) = File::Spec->splitpath($target); + my @target_path_elements = File::Spec->splitdir($target_path); + + # Add the dst_dirs to the dst root and join back together + $target_path = File::Spec->catdir( @target_root_elements, @target_path_elements ); + + # Now join it all together to get the complete path of the dest_file + $target = File::Spec->catpath( $target_root_volume, $target_path, $source_file ); + my $target_dir = File::Spec->catpath( $target_root_volume, $target_path, '' ); + + return $target_dir,$target; }; sub get_md5_of_non_flac_file { @@ -527,60 +310,73 @@ sub get_md5_of_non_flac_file { return $md5_code; }; +# use parallel processing to launch multiple transcoding processes sub path_and_conversion{ - # When "$transcode_enabled" is false it means that we are - # checking whether the file should be transcoded. If so, we - # return, and no further processing is taking place during this call. - # If the file is not to be transcoded, we stay and update tags if - # necessary. - - my $source = shift; - my $target = $flac_mp3_files{$source}; - my $transcode_enabled = shift; - my @messages = (); - my $t0; - my $elapsed_time = undef; - - $Options{debug} && msg("source: '$source'"); - $Options{debug} && msg("target: '$target'"); + my $source = shift; - # Step 1: get tags from flac file - my $source_tags = read_flac_tags($source); + # remove $source_dir from front of $src_file + my $target = $source; + $target =~ s{\Q$source_root/\E}{}xms; - # Step 2: hash to hold tags that will be updated - my $tags_to_update = preprocess_flac_tags( $source_tags ); + # Get directories in target and put in an array + # Note: the filename is the source file name + my ( $target_volume, $target_path, $source_file ) = File::Spec->splitpath($target); + my @target_path_elements = File::Spec->splitdir($target_path); - # Step 3: Initialise file processing flags - my ($pflags, $mess) = examine_destfile_tags( $target, $tags_to_update ); - push @messages, @$mess; + # Add the dst_dirs to the dst root and join back together + $target_path = File::Spec->catdir( @target_root_elements, @target_path_elements ); + # Add volume for OSes that require it (MSWin etc.) + $target_path = File::Spec->catpath( $target_root_volume, $target_path, '' ); - if ( ( !$$pflags{exists} || $$pflags{md5} || $Options{force} ) - && !$Options{tagsonly} ) { - - # Return if this file would be transcoded - return 1 unless ($transcode_enabled); - - $t0 = time; - # Step 4: Transcode the file based on the processing flags - $mess = transcode_file( $source, $target, $pflags ); - push @messages, @$mess; - $elapsed_time = time - $t0; - }; + # Get the basename of the dst file + my ( $target_base, $target_dir, $source_ext ) = fileparse( $source_file, qr{\Q.flac\E$}xmsi ); - # Step 5: Write the tags based on the processing flags - $mess = write_tags( $target, $tags_to_update, $pflags ); - push @messages, @$mess; + # Now join it all together to get the complete path of the dest_file + $target = File::Spec->catpath( $target_volume, $target_path, $target_base . '.mp3' ); - if (defined $elapsed_time) { - push @messages, sprintf("Conversion took %5.2f seconds.", $elapsed_time); - }; - - return \@messages; + convert_file( $source, $target ); }; +1; + +sub find_files { + my $path = shift; + my $regex = shift; + + my @found_files; + + my $found_list = File::Find::Rule->extras( { follow => 1 } )->name($regex); + if ( $Options{skipfile} ) { + my $skip_list = File::Find::Rule->directory->exec( + sub { + my ( $fname, $fpath, $frpath ) = @_; + if ( -f File::Spec->catdir( $frpath, $Options{skipfilename} ) ) { + return 1; + } + else { + return 0; + } + } + )->prune->discard; + @found_files = sort File::Find::Rule->or( $skip_list, $found_list )->in($path); + } + else { + + @found_files = sort $found_list ->in($path); + } + + $Options{debug} && msg( Dumper(@found_files) ); + + if ( $Options{info} ) { + my $file_count = scalar @found_files; + msg( "Found $file_count flac file" . ( $file_count > 1 ? 's' : '' . "\n" ) ); + } + + return \@found_files; +} + sub showusage { print <<"EOT"; - Usage: $0 [--pretend] [--quiet] [--debug] [--tagsonly] [--force] [--tagdiff] [--noskipfile] [--skipfilename=] [--lameargs='parameter-list'] --pretend Don't actually do anything @@ -590,32 +386,46 @@ sub showusage { --force Force transcoding and tag update even if not required --tagdiff Print source/dest tag values if different --lameargs='s' specify parameter(string) to be passed to the LAME Encoder - Default: "--noreplaygain -V 2 -h --nohist --quiet" + Default: "--noreplaygain --vbr-new -V 2 -h --nohist --quiet" --noskipfile Ignore any skip files --skipfilename Specify the name of the skip file. Default: flac2mp3.ignore - --processes=n Launch n parallel transcoding processes (limited support on Windows platform) + --processes=n Launch n parallel transcoding processes (does not work on Windows platform) Use with multi-core CPUs. Default: 1 --tagseparator=s Use "s" as the separator to join multiple instances of the same tag. Default: "/" --copyfiles Copy non-flac files to dest directories - --delete Delete surplus files and directories in destination, keeping in sync with source dir - -Example: $0 --processes=6 --delete ~/FLAC/ ~/MP3/ - EOT exit 0; } -sub msg { - print "@_\n" +sub msg { + my $msg = shift; + print "$msg\n"; } -sub msg_info { - # display only if "--quiet" option is not in use - $Options{info} && msg(@_) +sub convert_file { + my ( $source, $target ) = @_; + + $Options{debug} && msg("source: '$source'"); + $Options{debug} && msg("target: '$target'"); + + # get tags from flac file + my $source_tags = read_flac_tags($source); + + # hash to hold tags that will be updated + my $tags_to_update = preprocess_flac_tags( $source_tags ); + + # Initialise file processing flags + my $pflags = examine_destfile_tags( $target, $tags_to_update ); + + # Transcode the file based on the processing flags + transcode_file( $source, $target, $pflags ); + + # Write the tags based on the processing flags + write_tags( $target, $tags_to_update, $pflags ); } sub read_flac_tags { @@ -704,7 +514,6 @@ sub examine_destfile_tags { my $destfilename = shift; my $frames_ref = shift; my %frames_to_update = %$frames_ref; # this is only to minimize changes - my @return_messages = (); # Initialise file processing flags my %pflags = ( @@ -814,12 +623,11 @@ sub examine_destfile_tags { if ( $dest_text ne $srcframe ) { $pflags{tags} = 1; if ( $Options{tagdiff} ) { - push @return_messages, ( - "frame: '$frame'", - "srcframe value: '$srcframe'", - "destframe value: '$dest_text'" - ); + msg("frame: '$frame'"); + msg("srcframe value: '$srcframe'"); + msg("destframe value: '$dest_text'"); } + } } } @@ -841,7 +649,7 @@ sub examine_destfile_tags { msg( Dumper \%frames_to_update ); } - return \%pflags, \@return_messages; + return \%pflags; } sub transcode_file { @@ -849,92 +657,93 @@ sub transcode_file { my $target = shift; my $pflags_ref = shift; my %pflags = %$pflags_ref; # this is only to minimize changes - my @return_messages = (); - my $elapsed_time; - - # Transcode to a temp file in the destdir. - # Rename the file if the conversion completes sucessfully - # This avoids leaving incomplete files in the destdir - # If we're "pretending", don't create a File::Temp object - my $tmpfilename; - my $tmpfh; - if ( $Options{pretend} ) { - $tmpfilename = $target; - } - else { - # retrieve destination directory name - (undef, my $dst_dir) = File::Basename::fileparse($target); - # Create the destination directory if it - # doesn't already exist - unless (-d $dst_dir) { - # If necessary, allow a second check. Don't die just because the - # dir was created by another child (race condition): - mkpath($dst_dir) or (-d $dst_dir) - or die "Can't create directory $dst_dir\n"; - }; - $tmpfh = new File::Temp( - UNLINK => 1, - DIR => $dst_dir, - SUFFIX => '.tmp' - ); - $tmpfilename = $tmpfh->filename; - } - # Save message to be displayed on screen - push @return_messages, $pretendString . "Transcoding \"$source\"" ; - - my $convert_command = - "\"$flaccmd\" @flacargs \"$source\"" . "| \"$lamecmd\" @lameargs - \"$tmpfilename\""; + my ( $target_volume, $target_dir, $target_filename ) = File::Spec->splitpath($target); + my $dst_dir = File::Spec->catpath( $target_volume, $target_dir, '' ); + + if ( ( !$pflags{exists} || $pflags{md5} || $Options{force} ) + && !$Options{tagsonly} ) + { + + # Transcode to a temp file in the destdir. + # Rename the file if the conversion completes sucessfully + # This avoids leaving incomplete files in the destdir + # If we're "pretending", don't create a File::Temp object + my $tmpfilename; + my $tmpfh; + if ( $Options{pretend} ) { + $tmpfilename = $target; + } + else { + + # Create the destination directory if it + # doesn't already exist + unless (-d $dst_dir) { + # If necessary, allow a second check. Don't die just because the + # dir was created by another child (race condition): + mkpath($dst_dir) or (-d $dst_dir) + or die "Can't create directory $dst_dir\n"; + }; + $tmpfh = new File::Temp( + UNLINK => 1, + DIR => $dst_dir, + SUFFIX => '.tmp' + ); + $tmpfilename = $tmpfh->filename; + } + $Options{info} + && msg( $pretendString . "Transcoding \"$source\"" ); + + my $convert_command = "\"$flaccmd\" @flacargs \"$source\"" . "| \"$lamecmd\" @lameargs - \"$tmpfilename\""; - $Options{debug} && msg("transcode: $convert_command"); + $Options{debug} && msg("transcode: $convert_command"); - # Convert the file (unless we're pretending} - my $exit_value; - if ( !$Options{pretend} ) { - $exit_value = system($convert_command); - } - else { - $exit_value = 0; - } + # Convert the file (unless we're pretending} + my $exit_value; + if ( !$Options{pretend} ) { + $exit_value = system($convert_command); + } + else { + $exit_value = 0; + } - $Options{debug} - && msg("Exit value from convert command: $exit_value"); + $Options{debug} + && msg("Exit value from convert command: $exit_value"); - if ($exit_value) { - msg("$convert_command failed with exit code $exit_value"); + if ($exit_value) { + msg("$convert_command failed with exit code $exit_value"); - # delete the destfile if it exists - unlink $tmpfilename; + # delete the destfile if it exists + unlink $tmpfilename; - # should check exit status of this command - exit($exit_value); - } + # should check exit status of this command - if ( !$Options{pretend} ) { + exit($exit_value); + } - # If we get here, assume the conversion has succeeded - $tmpfh->unlink_on_destroy(0); - $tmpfh->close; - croak "Failed to rename '$tmpfilename' to '$target' $!" - unless rename( $tmpfilename, $target ); + if ( !$Options{pretend} ) { - # the destfile now exists! - $pflags{exists} = 1; + # If we get here, assume the conversion has succeeded + $tmpfh->unlink_on_destroy(0); + $tmpfh->close; + croak "Failed to rename '$tmpfilename' to '$target' $!" + unless rename( $tmpfilename, $target ); - # and the tags need writing - $pflags{tags} = 1; - } + # the destfile now exists! + $pflags{exists} = 1; + # and the tags need writing + $pflags{tags} = 1; + } + } if ( $Options{debug} ) { msg("pf_exists: $pflags{exists}"); msg("pf_tags: $pflags{tags}"); msg( "\$Options{pretend}: " . ( $Options{pretend} ? 'set' : 'not set' ) ); - } - + } + %$pflags_ref = %pflags; # this is only to minimize changes - - return \@return_messages; } sub write_tags { @@ -943,13 +752,16 @@ sub write_tags { my $pflags_ref = shift; my %frames_to_update = %$frames_ref; # this is only to minimize changes my %pflags = %$pflags_ref; # this is only to minimize changes - my @return_messages = (); # Write the tags - if ( $pflags{exists} && ( $pflags{tags} || $Options{force} ) ) { + if ($pflags{exists} + && ( $pflags{tags} + || $Options{force} ) + ) + { - # save message to be displayed on screen - push @return_messages, $pretendString . "Writing tags to \"$destfilename\""; + $Options{info} + && msg( $pretendString . "Writing tags to \"$destfilename\"" ); if ( !$Options{pretend} ) { my $mp3 = MP3::Tag->new($destfilename); @@ -1012,7 +824,6 @@ sub write_tags { # utime $srcstat->mtime, $srcstat->mtime, $destfilename; } } - return \@return_messages; } sub INT_Handler { @@ -1038,7 +849,8 @@ sub fixUpTrackNumber { $trackNum = sprintf( "%02u", $trackNum ); } else { - msg_info('TRACKNUMBER not numeric'); + $Options{info} + && msg('TRACKNUMBER not numeric'); } } return $trackNum; @@ -1088,4 +900,4 @@ sub picsToAPICframes { # vim:set softtabstop=4: # vim:set shiftwidth=4: -__END__ \ No newline at end of file +__END__ diff --git a/lib/MP3/Tag/ID3v2.pm b/lib/MP3/Tag/ID3v2.pm index e7e3934..d438576 100755 --- a/lib/MP3/Tag/ID3v2.pm +++ b/lib/MP3/Tag/ID3v2.pm @@ -638,8 +638,7 @@ sub insert_space { # Change file permissions to default value. # Permissions will otherwise (at least on *nix) - # be 0600 regardless of umask, due a peculiar choice choice - # by the developers of the File::Temp module. + # be 0600 regardless of umask, due the File::Temp module. chmod umask ^ 0666, $tempfile; if ($@) { From 37e3345c4eb5f048658c62adc7d2ac048b55f045 Mon Sep 17 00:00:00 2001 From: Carl Asplund Date: Sun, 10 Mar 2013 00:36:53 +0100 Subject: [PATCH 06/15] removing obsolete LAME parameter --- flac2mp3.pl | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/flac2mp3.pl b/flac2mp3.pl index da4099c..ed250a7 100755 --- a/flac2mp3.pl +++ b/flac2mp3.pl @@ -53,7 +53,6 @@ # Modify lame options if required my @lameargs = qw ( --noreplaygain - --vbr-new -V 2 -h --nohist @@ -386,7 +385,7 @@ sub showusage { --force Force transcoding and tag update even if not required --tagdiff Print source/dest tag values if different --lameargs='s' specify parameter(string) to be passed to the LAME Encoder - Default: "--noreplaygain --vbr-new -V 2 -h --nohist --quiet" + Default: "--noreplaygain -V 2 -h --nohist --quiet" --noskipfile Ignore any skip files --skipfilename Specify the name of the skip file. Default: flac2mp3.ignore From 7b19adcac84e9cb0a7a4d2e0ada1e1cd771f9a71 Mon Sep 17 00:00:00 2001 From: Carl Asplund Date: Sun, 10 Mar 2013 19:30:42 +0100 Subject: [PATCH 07/15] Turn off id3 v2.3 unsyncing - again. This time due to inability of Logitech Media Sever to cope with it. --- flac2mp3.pl | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/flac2mp3.pl b/flac2mp3.pl index ed250a7..b28e39f 100755 --- a/flac2mp3.pl +++ b/flac2mp3.pl @@ -193,6 +193,10 @@ $Options{info} && msg("Using $cmd from: $cmdpath"); } +# Turn off unsyncing, due to broken implementation in +# some software (such as Logitech Media Server) +MP3::Tag->config(id3v23_unsync => 0); + # Convert directories to absolute paths $source_root = File::Spec->rel2abs($source_root); $target_root = File::Spec->rel2abs($target_root); From 930f975c31576a271b88d84cf5c8ad974364940c Mon Sep 17 00:00:00 2001 From: Carl Asplund Date: Sun, 10 Mar 2013 22:42:03 +0100 Subject: [PATCH 08/15] msg_info - new sub for displaying messages Define a new subroutine msg_info which prints messages to screen only if option{info} is true. No need to use the check to info flag ( $Options{info} && msg(...) ) every time something is to be displayed. Also fix a few bugs, where message output was not properly suppressed with the --quiet option set. --- flac2mp3.pl | 53 +++++++++++++++++++++++------------------------------ 1 file changed, 23 insertions(+), 30 deletions(-) diff --git a/flac2mp3.pl b/flac2mp3.pl index b28e39f..af83846 100755 --- a/flac2mp3.pl +++ b/flac2mp3.pl @@ -190,7 +190,7 @@ $cmdpath = which($cmd); } croak "$cmd not found" unless $cmdpath; - $Options{info} && msg("Using $cmd from: $cmdpath"); + msg_info("Using $cmd from: $cmdpath"); } # Turn off unsyncing, due to broken implementation in @@ -210,8 +210,7 @@ # Will need to only count files that are going to be processed. # Hmmm could get complicated. -$Options{info} - && msg( $pretendString . "Processing directory: $source_root" ); +msg_info( $pretendString . "Processing directory: $source_root" ); # Now look for files in the source dir # (following symlinks) @@ -223,7 +222,7 @@ my @target_root_elements = File::Spec->splitdir($target_root_path); # use parallel processing to launch multiple transcoding processes -msg("Using $Options{processes} transcoding processes.\n"); +msg_info("Using $Options{processes} transcoding processes.\n"); my $pm = new Parallel::ForkManager($Options{processes}); foreach my $src_file (@flac_files) { $pm->start and next; # Forks and returns the pid for the child @@ -238,8 +237,7 @@ = sort File::Find::Rule->file()->extras( { follow => 1 } )->not_name(qr/\.flac$/i) ->in($source_root); my $non_flac_file_count = scalar @non_flac_files; - $Options{info} && - msg( "Found $non_flac_file_count non-flac file" .( $non_flac_file_count != 1 ? 's' : '' . "\n" ) ); + msg_info( "Found $non_flac_file_count non-flac file" .( $non_flac_file_count != 1 ? 's' : '' . "\n" ) ); # Copy non-flac files from source to dest directories my $t0 = time; @@ -273,12 +271,14 @@ }; $cntr_all ++; # Show the progress every second - if ( ((time - $t0) >= 1) || ($cntr_all==$non_flac_file_count) ) { + if ( $Options{info} && + ( ((time - $t0) >= 1) || ($cntr_all==$non_flac_file_count) ) ) { $t0 = time; - print("\r" . $pretendString . $cntr_copied . " non-flac files of " . $cntr_all ." were copied to dest directories."); + print("\r" . $pretendString . $cntr_copied . + " non-flac files of " . $cntr_all ." were copied to dest directories."); }; }; - msg("\n"); # double line feed + msg_info("\n"); # double line feed }; @@ -313,7 +313,6 @@ sub get_md5_of_non_flac_file { return $md5_code; }; -# use parallel processing to launch multiple transcoding processes sub path_and_conversion{ my $source = shift; @@ -370,10 +369,8 @@ sub find_files { $Options{debug} && msg( Dumper(@found_files) ); - if ( $Options{info} ) { - my $file_count = scalar @found_files; - msg( "Found $file_count flac file" . ( $file_count > 1 ? 's' : '' . "\n" ) ); - } + my $file_count = scalar @found_files; + msg_info( "Found $file_count flac file" . ( $file_count > 1 ? 's' : '' . "\n" ) ); return \@found_files; } @@ -404,9 +401,13 @@ sub showusage { exit 0; } -sub msg { - my $msg = shift; - print "$msg\n"; +sub msg { + print "@_\n" +} + +sub msg_info { + # display only if "--quiet" option is not in use + $Options{info} && msg(@_) } sub convert_file { @@ -630,7 +631,6 @@ sub examine_destfile_tags { msg("srcframe value: '$srcframe'"); msg("destframe value: '$dest_text'"); } - } } } @@ -694,8 +694,7 @@ sub transcode_file { ); $tmpfilename = $tmpfh->filename; } - $Options{info} - && msg( $pretendString . "Transcoding \"$source\"" ); + msg_info( $pretendString . "Transcoding \"$source\"" ); my $convert_command = "\"$flaccmd\" @flacargs \"$source\"" . "| \"$lamecmd\" @lameargs - \"$tmpfilename\""; @@ -757,14 +756,9 @@ sub write_tags { my %pflags = %$pflags_ref; # this is only to minimize changes # Write the tags - if ($pflags{exists} - && ( $pflags{tags} - || $Options{force} ) - ) - { + if ( $pflags{exists} && ( $pflags{tags} || $Options{force} ) ) { - $Options{info} - && msg( $pretendString . "Writing tags to \"$destfilename\"" ); + msg_info( $pretendString . "Writing tags to \"$destfilename\"" ); if ( !$Options{pretend} ) { my $mp3 = MP3::Tag->new($destfilename); @@ -852,8 +846,7 @@ sub fixUpTrackNumber { $trackNum = sprintf( "%02u", $trackNum ); } else { - $Options{info} - && msg('TRACKNUMBER not numeric'); + msg_info('TRACKNUMBER not numeric'); } } return $trackNum; @@ -903,4 +896,4 @@ sub picsToAPICframes { # vim:set softtabstop=4: # vim:set shiftwidth=4: -__END__ +__END__ \ No newline at end of file From ff28ee2422edfe1d235886cef250de7dc42efd17 Mon Sep 17 00:00:00 2001 From: Carl Asplund Date: Sun, 10 Mar 2013 23:14:29 +0100 Subject: [PATCH 09/15] fix bug where dir could be created despite --pretend option With both --copy and --pretend options selected a directory could sometimes be created in destination (it shouldn't be possible). --- flac2mp3.pl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/flac2mp3.pl b/flac2mp3.pl index af83846..009ba7a 100755 --- a/flac2mp3.pl +++ b/flac2mp3.pl @@ -257,11 +257,11 @@ }; } else { - # Create the destination directory if it - # doesn't already exist - mkpath($dst_dir) - or die "Can't create directory $dst_dir\n" - unless -d $dst_dir; + # Create the destination directory if it + # doesn't already exist + unless ( $Options{pretend} || -d $dst_dir ) { + mkpath($dst_dir) or die "Can't create directory $dst_dir\n"; + } }; if ( $do_copy ) { unless ( $Options{pretend} ) { From 9570eaddb2c5eec26c66985aa92e579b4e8846f2 Mon Sep 17 00:00:00 2001 From: Carl Asplund Date: Sun, 10 Mar 2013 23:44:41 +0100 Subject: [PATCH 10/15] windows lame/flac paths As a service to windows users, the paths to typical locations of lame and flac are provided. These will work for most (intended for people who haven't included these paths into the envionment variables). --- flac2mp3.pl | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/flac2mp3.pl b/flac2mp3.pl index 009ba7a..1951901 100755 --- a/flac2mp3.pl +++ b/flac2mp3.pl @@ -47,8 +47,18 @@ # c:/windows/system32/flac.exe # or # c:\\windows\\system32\\flac.exe -my $flaccmd = 'flac'; -my $lamecmd = 'lame'; + +my ($flaccmd, $lamecmd); +if ($^O eq 'MSWin32' or $^O eq 'MSWin64') { + # This is an example of typical Windows filepaths. + # Change them if necessary: + $flaccmd = q|C:\Program Files (x86)\FLAC\flac.exe|; + $lamecmd = q|C:\Program Files\LAME\lame.exe|; +} +else { + $flaccmd = 'flac'; + $lamecmd = 'lame'; +} # Modify lame options if required my @lameargs = qw ( From a3fd80085b3d95da6d8db8b63f109acdbf81c261 Mon Sep 17 00:00:00 2001 From: Carl Asplund Date: Mon, 11 Mar 2013 23:57:28 +0100 Subject: [PATCH 11/15] refactor code for file find and file paths Functionality is unchanged, but the way source and target file paths are retrieved is more generalized. This is an intermediary step towards the implementation of the "--delete" option. --- flac2mp3.pl | 147 ++++++++++++++++++++++++---------------------------- 1 file changed, 69 insertions(+), 78 deletions(-) diff --git a/flac2mp3.pl b/flac2mp3.pl index 1951901..57aa459 100755 --- a/flac2mp3.pl +++ b/flac2mp3.pl @@ -224,12 +224,12 @@ # Now look for files in the source dir # (following symlinks) +my %flac_mp3_files = get_all_paths('name', '.flac', $source_root, $target_root, '.mp3'); +my @flac_files = sort keys { %flac_mp3_files }; + +my $file_count = scalar @flac_files; +msg_info( "Found $file_count flac file" . ( $file_count > 1 ? 's' : '' . "\n" ) ); -my @flac_files = @{ find_files( $source_root, qr/\.flac$/i ) }; - -# Get directories from target_dir and put in an array -my ( $target_root_volume, $target_root_path, $target_root_file ) = File::Spec->splitpath( $target_root, 1 ); -my @target_root_elements = File::Spec->splitdir($target_root_path); # use parallel processing to launch multiple transcoding processes msg_info("Using $Options{processes} transcoding processes.\n"); @@ -243,9 +243,8 @@ if ( $Options{copyfiles} ) { - my @non_flac_files - = sort File::Find::Rule->file()->extras( { follow => 1 } )->not_name(qr/\.flac$/i) - ->in($source_root); + my %non_flac_files = get_all_paths('not_name', '.flac', $source_root, $target_root, ''); + my @non_flac_files = keys %non_flac_files; my $non_flac_file_count = scalar @non_flac_files; msg_info( "Found $non_flac_file_count non-flac file" .( $non_flac_file_count != 1 ? 's' : '' . "\n" ) ); @@ -254,7 +253,7 @@ my $cntr_all = 0; my $cntr_copied = 0; foreach my $src_file (@non_flac_files) { - my ($dst_dir, $dst_file) = get_dest_file_path_non_flac($src_file); + my $dst_file = $non_flac_files{$src_file}; # Flag which determines if file should be copied: my $do_copy = 1; # Don't copy file if it already exists in dest directory and @@ -269,6 +268,8 @@ else { # Create the destination directory if it # doesn't already exist + (undef, my $dst_dir) = + File::Basename::fileparse($dst_file); # retrieve directory name unless ( $Options{pretend} || -d $dst_dir ) { mkpath($dst_dir) or die "Can't create directory $dst_dir\n"; } @@ -291,29 +292,6 @@ msg_info("\n"); # double line feed }; - -sub get_dest_file_path_non_flac { - my $source = shift; - - # remove $source_dir from front of $src_file - my $target = $source; - $target =~ s{\Q$source_root/\E}{}xms; - - # Get directories in target and put in an array - # Note: the filename is the source file name - my ( $target_volume, $target_path, $source_file ) = File::Spec->splitpath($target); - my @target_path_elements = File::Spec->splitdir($target_path); - - # Add the dst_dirs to the dst root and join back together - $target_path = File::Spec->catdir( @target_root_elements, @target_path_elements ); - - # Now join it all together to get the complete path of the dest_file - $target = File::Spec->catpath( $target_root_volume, $target_path, $source_file ); - my $target_dir = File::Spec->catpath( $target_root_volume, $target_path, '' ); - - return $target_dir,$target; -}; - sub get_md5_of_non_flac_file { my $file = shift; open(FILE, $file) or die "Can't open '$file': $!"; @@ -323,42 +301,56 @@ sub get_md5_of_non_flac_file { return $md5_code; }; -sub path_and_conversion{ - my $source = shift; - - # remove $source_dir from front of $src_file - my $target = $source; - $target =~ s{\Q$source_root/\E}{}xms; - - # Get directories in target and put in an array - # Note: the filename is the source file name - my ( $target_volume, $target_path, $source_file ) = File::Spec->splitpath($target); - my @target_path_elements = File::Spec->splitdir($target_path); - - # Add the dst_dirs to the dst root and join back together - $target_path = File::Spec->catdir( @target_root_elements, @target_path_elements ); - # Add volume for OSes that require it (MSWin etc.) - $target_path = File::Spec->catpath( $target_root_volume, $target_path, '' ); - - # Get the basename of the dst file - my ( $target_base, $target_dir, $source_ext ) = fileparse( $source_file, qr{\Q.flac\E$}xmsi ); - - # Now join it all together to get the complete path of the dest_file - $target = File::Spec->catpath( $target_volume, $target_path, $target_base . '.mp3' ); - - convert_file( $source, $target ); -}; - -1; - -sub find_files { - my $path = shift; - my $regex = shift; +sub get_all_dirs { + my ($root, $new_root) = @_; + # we supply no suffix, so we search for directories (not files): + my @orig_dirs = @{ find_files_or_dirs($root) }; + + my @dirs = (); + foreach my $dir (@orig_dirs) { + # strip source root dir from path... + my $rel_path = File::Spec->abs2rel( $dir, $root ); + # then replace it with target root dir + push @dirs, File::Spec->rel2abs( $rel_path, $new_root ); + } + return sort @dirs; +} - my @found_files; +sub get_all_paths { + my ($rule, $suffix, $root, $new_root, $new_suffix) = @_; + my @orig_files = @{ find_files_or_dirs($root, $rule, $suffix) }; + + # Even if $root = $new_root, we need to do the following operations + # to get a consistent path format (otherwise problematic in e.g. MS Win) + # that is suitable for later string comparison: + my %paths = (); + foreach my $src (@orig_files) { + # Strip source root dir from file path + my $rel_path = File::Spec->abs2rel( $src, $root ); + # ... then replace it with target root dir and change file suffix. + ($paths{$src} = File::Spec->rel2abs( $rel_path, $new_root ) ) =~ s{$suffix$}{$new_suffix}xmsi; + } + return %paths +} - my $found_list = File::Find::Rule->extras( { follow => 1 } )->name($regex); - if ( $Options{skipfile} ) { +sub find_files_or_dirs { + my $path = shift; + my $rule = shift; + my $suffix = shift; + + # If a matching rule and file suffix is defined we are looking for files, + # otherwise we are looking for directories. + my $found_list; + if (defined $rule && defined $suffix) { + $found_list = File::Find::Rule->file()->extras( { follow => 1 } )->$rule(qr{$suffix$}xmsi) + } + else { + $found_list = File::Find::Rule->directory->extras( { follow => 1 } ) + }; + + # skip any directories where a "skipfile" is found + my @found; + if ( $Options{skipfile} && ($path eq $source_root) ) { my $skip_list = File::Find::Rule->directory->exec( sub { my ( $fname, $fpath, $frpath ) = @_; @@ -370,20 +362,20 @@ sub find_files { } } )->prune->discard; - @found_files = sort File::Find::Rule->or( $skip_list, $found_list )->in($path); + @found = sort File::Find::Rule->or( $skip_list, $found_list )->in($path); } else { - - @found_files = sort $found_list ->in($path); + @found = sort $found_list ->in($path); } +return \@found; +} - $Options{debug} && msg( Dumper(@found_files) ); - - my $file_count = scalar @found_files; - msg_info( "Found $file_count flac file" . ( $file_count > 1 ? 's' : '' . "\n" ) ); +sub path_and_conversion{ + my $source = shift; + my $target = $flac_mp3_files{$source}; - return \@found_files; -} + convert_file( $source, $target ); +}; sub showusage { print <<"EOT"; @@ -671,9 +663,6 @@ sub transcode_file { my $pflags_ref = shift; my %pflags = %$pflags_ref; # this is only to minimize changes - my ( $target_volume, $target_dir, $target_filename ) = File::Spec->splitpath($target); - my $dst_dir = File::Spec->catpath( $target_volume, $target_dir, '' ); - if ( ( !$pflags{exists} || $pflags{md5} || $Options{force} ) && !$Options{tagsonly} ) { @@ -688,6 +677,8 @@ sub transcode_file { $tmpfilename = $target; } else { + # retrieve destination directory name + (undef, my $dst_dir) = File::Basename::fileparse($target); # Create the destination directory if it # doesn't already exist From 3d0758d0445e41e542137ab15875792359c71276 Mon Sep 17 00:00:00 2001 From: Carl Asplund Date: Tue, 12 Mar 2013 00:49:42 +0100 Subject: [PATCH 12/15] add option --delete This option works as the --delete option in "rsync": it deletes surplus files and directories in the destination, keeping in perfect sync with the source dir. --- flac2mp3.pl | 65 ++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) diff --git a/flac2mp3.pl b/flac2mp3.pl index 57aa459..c753c5c 100755 --- a/flac2mp3.pl +++ b/flac2mp3.pl @@ -162,7 +162,7 @@ GetOptions( \%Options, "quiet!", "tagdiff", "debug!", "tagsonly!", "force!", "usage", "help", "version", "pretend", "skipfile!", "skipfilename=s", - "processes=i", "tagseparator=s", "lameargs=s", "copyfiles" + "processes=i", "tagseparator=s", "lameargs=s", "copyfiles", "delete" ); # info flag is the inverse of --quiet @@ -230,6 +230,10 @@ my $file_count = scalar @flac_files; msg_info( "Found $file_count flac file" . ( $file_count > 1 ? 's' : '' . "\n" ) ); +# If allowed, delete surplus files and folders from target directory, keeping +# it in perfect sync with (i.e. a mirror of) the source directory. +delete_excess_files_from_dest($source_root, $target_root) if ( $Options{delete} ) ; + # use parallel processing to launch multiple transcoding processes msg_info("Using $Options{processes} transcoding processes.\n"); @@ -301,6 +305,64 @@ sub get_md5_of_non_flac_file { return $md5_code; }; +sub delete_excess_files_from_dest { + my ($source_root, $target_root) = @_; + + # Generate (source => target) hashes for the files found using + # each of the following combinations of root dirs and file suffixes + my %existing_target_mp3_files = get_all_paths('name', '.mp3', $target_root, $target_root, '.mp3'); + my %existing_source_mp3_files = get_all_paths('name', '.mp3', $source_root, $target_root, '.mp3'); + my %non_flac_files = get_all_paths('not_name', '.flac', $source_root, $target_root, ''); + my %existing_target_non_mp3_files = get_all_paths('not_name', '.mp3', $target_root, $target_root, ''); + + # 1. calculate what files to expect in directory after finished transcoding and copying + my @expected_transcoded_mp3s = keys { reverse %flac_mp3_files }; # expected mp3 files in target from transcoded flac files + my @expected_copied_files = keys { reverse %non_flac_files }; # expected files in target copied from non-flac files in source + my @expected_files = uniq(@expected_transcoded_mp3s, @expected_copied_files); # Join the arrays and remove any duplicates + + # 2. check what files are actually present + my @actual_mp3s = keys { reverse %existing_target_mp3_files }; # actual existing mp3 files in target + my @actual_non_mp3s = keys { reverse %existing_target_non_mp3_files }; # existing non-mp3 files in target + my @actual_files = (@actual_mp3s, @actual_non_mp3s); # Join the arrays (being mutually exclusive, there is no overlap) + + # 3. determine which files to remove from target directory tree + my @files_to_remove = single_difference(\@expected_files, \@actual_files); + + # 4. determine which subdirectories to remove from target directory tree + my @expected_subdirs_in_target = get_all_dirs($source_root,$target_root); + my @actual_subdirs_in_target = get_all_dirs($target_root,$target_root); + my @dirs_to_remove = single_difference(\@expected_subdirs_in_target, \@actual_subdirs_in_target); + + # 5. carry out the deletions + foreach my $file (@files_to_remove) { + $Options{pretend} || unlink $file or die "Unable to delete $file: $!"; + msg_info($pretendString . "Deleted \"$file\""); + } + foreach my $dir (reverse sort @dirs_to_remove) { + $Options{pretend} || File::Path->remove_tree($dir) or die "Unable to delete directory $dir: $!"; + msg_info($pretendString . "Deleted directory \"$dir\""); + } +} + +# Return all unique elements of input array @_ +sub uniq { + return sort keys %{{ map { $_ => 1 } @_ }} +}; + +# Acccept two arrays @A and @B as argument, return elements in @B that aren't in @A. +sub single_difference { + my ($A, $B) = @_; + + # build lookup table + my %seen = (); + my @bonly = (); + @seen{@$A} = (1) x @$A; + foreach my $item (@$B) { + push(@bonly, $item) unless $seen{$item}; + } + return sort @bonly; +} + sub get_all_dirs { my ($root, $new_root) = @_; # we supply no suffix, so we search for directories (not files): @@ -399,6 +461,7 @@ sub showusage { same tag. Default: "/" --copyfiles Copy non-flac files to dest directories + --delete Delete surplus files and directories in destination, keeping in sync with source dir EOT exit 0; } From c3670133b503fe7352e3e7b83e7f5a49f8d0f0c5 Mon Sep 17 00:00:00 2001 From: Carl Asplund Date: Wed, 13 Mar 2013 23:57:07 +0100 Subject: [PATCH 13/15] move code for option --copyfiles to subroutine Improve readabiliy of the main program by moving the code for (non-flac) file copying to a subroutine. --- flac2mp3.pl | 124 ++++++++++++++++++++++++++++------------------------ 1 file changed, 68 insertions(+), 56 deletions(-) diff --git a/flac2mp3.pl b/flac2mp3.pl index c753c5c..ae38445 100755 --- a/flac2mp3.pl +++ b/flac2mp3.pl @@ -246,64 +246,14 @@ $pm->wait_all_children; -if ( $Options{copyfiles} ) { - my %non_flac_files = get_all_paths('not_name', '.flac', $source_root, $target_root, ''); - my @non_flac_files = keys %non_flac_files; - my $non_flac_file_count = scalar @non_flac_files; - msg_info( "Found $non_flac_file_count non-flac file" .( $non_flac_file_count != 1 ? 's' : '' . "\n" ) ); +# If allowed, copy non-flac files to destination dirs +copy_non_flacs($source_root, $target_root) if ( $Options{copyfiles} ); - # Copy non-flac files from source to dest directories - my $t0 = time; - my $cntr_all = 0; - my $cntr_copied = 0; - foreach my $src_file (@non_flac_files) { - my $dst_file = $non_flac_files{$src_file}; - # Flag which determines if file should be copied: - my $do_copy = 1; - # Don't copy file if it already exists in dest directory and - # has identical md5 to the source file - if (-e $dst_file) { - my $src_md5 = get_md5_of_non_flac_file($src_file); - my $dst_md5 = get_md5_of_non_flac_file($dst_file); - if ($src_md5 eq $dst_md5) { - $do_copy = 0; # Don't copy if equal md5 - }; - } - else { - # Create the destination directory if it - # doesn't already exist - (undef, my $dst_dir) = - File::Basename::fileparse($dst_file); # retrieve directory name - unless ( $Options{pretend} || -d $dst_dir ) { - mkpath($dst_dir) or die "Can't create directory $dst_dir\n"; - } - }; - if ( $do_copy ) { - unless ( $Options{pretend} ) { - copy($src_file,$dst_file) || die("Can't copy this FILE: $src_file !"); - } - $cntr_copied ++; - }; - $cntr_all ++; - # Show the progress every second - if ( $Options{info} && - ( ((time - $t0) >= 1) || ($cntr_all==$non_flac_file_count) ) ) { - $t0 = time; - print("\r" . $pretendString . $cntr_copied . - " non-flac files of " . $cntr_all ." were copied to dest directories."); - }; - }; - msg_info("\n"); # double line feed -}; +1; +# ------------ Main program ends here -------------------------------------- -sub get_md5_of_non_flac_file { - my $file = shift; - open(FILE, $file) or die "Can't open '$file': $!"; - binmode(FILE); - my $md5_code = Digest::MD5->new->addfile(*FILE)->hexdigest; - close FILE; - return $md5_code; -}; + +# ------------ Subroutines start here -------------------------------------- sub delete_excess_files_from_dest { my ($source_root, $target_root) = @_; @@ -432,6 +382,68 @@ sub find_files_or_dirs { return \@found; } +sub copy_non_flacs { + my ($source_root, $target_root) = @_; + + my %non_flac_files = get_all_paths('not_name', '.flac', $source_root, $target_root, ''); + my @non_flac_files = keys %non_flac_files; + my $non_flac_file_count = scalar @non_flac_files; + msg_info( "Found $non_flac_file_count non-flac file" . + ( $non_flac_file_count != 1 ? 's' : '' . "\n" ) ); + + # Copy non-flac files from source to dest directories + my $t0 = time; + my $cntr_all = 0; + my $cntr_copied = 0; + foreach my $src_file (@non_flac_files) { + my $dst_file = $non_flac_files{$src_file}; + # Flag which determines if file should be copied: + my $do_copy = 1; + # Don't copy file if it already exists in dest directory and + # has identical md5 to the source file + if (-e $dst_file) { + my $src_md5 = get_md5_of_non_flac_file($src_file); + my $dst_md5 = get_md5_of_non_flac_file($dst_file); + if ($src_md5 eq $dst_md5) { + $do_copy = 0; # Don't copy if equal md5 + }; + } + else { + # Create the destination directory if it + # doesn't already exist + (undef, my $dst_dir) = + File::Basename::fileparse($dst_file); # retrieve directory name + unless ( $Options{pretend} || -d $dst_dir ) { + mkpath($dst_dir) or die "Can't create directory $dst_dir\n"; + } + }; + if ( $do_copy ) { + unless ( $Options{pretend} ) { + copy($src_file,$dst_file) || die("Can't copy this FILE: $src_file !"); + } + $cntr_copied ++; + }; + $cntr_all ++; + # Show the progress every second + if ( $Options{info} && + ( ((time - $t0) >= 1) || ($cntr_all==$non_flac_file_count) ) ) { + $t0 = time; + print("\r" . $pretendString . $cntr_copied . + " non-flac files of " . $cntr_all ." were copied to dest directories."); + }; + }; + msg_info("\n"); # double line feed +}; + +sub get_md5_of_non_flac_file { + my $file = shift; + open(FILE, $file) or die "Can't open '$file': $!"; + binmode(FILE); + my $md5_code = Digest::MD5->new->addfile(*FILE)->hexdigest; + close FILE; + return $md5_code; +}; + sub path_and_conversion{ my $source = shift; my $target = $flac_mp3_files{$source}; From 5127849555a0844d1e0ffa9d00a064d37c192af2 Mon Sep 17 00:00:00 2001 From: Carl Asplund Date: Sun, 17 Mar 2013 22:41:52 +0100 Subject: [PATCH 14/15] refactor path_and_convert sub (step 1/5) In a first step towards progress reporting during transcoding, refactor code for the 'path_and_conversion' sub. The checking of flags that determine whether a file should be transcoded is lifted into this sub from 'transcode_file'. The latter sub therefore loses a wrapping if-statement and consequently also loses an indentation step, which makes the changeset look much messier than it actually is. The sub 'convert_file' is no longer used. --- flac2mp3.pl | 171 +++++++++++++++++++++++++--------------------------- 1 file changed, 83 insertions(+), 88 deletions(-) diff --git a/flac2mp3.pl b/flac2mp3.pl index ae38445..9a5a76b 100755 --- a/flac2mp3.pl +++ b/flac2mp3.pl @@ -448,7 +448,28 @@ sub path_and_conversion{ my $source = shift; my $target = $flac_mp3_files{$source}; - convert_file( $source, $target ); + $Options{debug} && msg("source: '$source'"); + $Options{debug} && msg("target: '$target'"); + + # Step 1: get tags from flac file + my $source_tags = read_flac_tags($source); + + # Step 2: hash to hold tags that will be updated + my $tags_to_update = preprocess_flac_tags( $source_tags ); + + # Step 3: Initialise file processing flags + my ($pflags) = examine_destfile_tags( $target, $tags_to_update ); + + + if ( ( !$$pflags{exists} || $$pflags{md5} || $Options{force} ) + && !$Options{tagsonly} ) { + + # Step 4: Transcode the file based on the processing flags + transcode_file( $source, $target, $pflags ); + }; + + # Step 5: Write the tags based on the processing flags + write_tags( $target, $tags_to_update, $pflags ); }; sub showusage { @@ -487,28 +508,6 @@ sub msg_info { $Options{info} && msg(@_) } -sub convert_file { - my ( $source, $target ) = @_; - - $Options{debug} && msg("source: '$source'"); - $Options{debug} && msg("target: '$target'"); - - # get tags from flac file - my $source_tags = read_flac_tags($source); - - # hash to hold tags that will be updated - my $tags_to_update = preprocess_flac_tags( $source_tags ); - - # Initialise file processing flags - my $pflags = examine_destfile_tags( $target, $tags_to_update ); - - # Transcode the file based on the processing flags - transcode_file( $source, $target, $pflags ); - - # Write the tags based on the processing flags - write_tags( $target, $tags_to_update, $pflags ); -} - sub read_flac_tags { my $source = shift; @@ -738,88 +737,84 @@ sub transcode_file { my $pflags_ref = shift; my %pflags = %$pflags_ref; # this is only to minimize changes - if ( ( !$pflags{exists} || $pflags{md5} || $Options{force} ) - && !$Options{tagsonly} ) - { - - # Transcode to a temp file in the destdir. - # Rename the file if the conversion completes sucessfully - # This avoids leaving incomplete files in the destdir - # If we're "pretending", don't create a File::Temp object - my $tmpfilename; - my $tmpfh; - if ( $Options{pretend} ) { - $tmpfilename = $target; - } - else { - # retrieve destination directory name - (undef, my $dst_dir) = File::Basename::fileparse($target); - - # Create the destination directory if it - # doesn't already exist - unless (-d $dst_dir) { - # If necessary, allow a second check. Don't die just because the - # dir was created by another child (race condition): - mkpath($dst_dir) or (-d $dst_dir) - or die "Can't create directory $dst_dir\n"; - }; - $tmpfh = new File::Temp( - UNLINK => 1, - DIR => $dst_dir, - SUFFIX => '.tmp' - ); - $tmpfilename = $tmpfh->filename; - } - msg_info( $pretendString . "Transcoding \"$source\"" ); + # Transcode to a temp file in the destdir. + # Rename the file if the conversion completes sucessfully + # This avoids leaving incomplete files in the destdir + # If we're "pretending", don't create a File::Temp object + my $tmpfilename; + my $tmpfh; + if ( $Options{pretend} ) { + $tmpfilename = $target; + } + else { + # retrieve destination directory name + (undef, my $dst_dir) = File::Basename::fileparse($target); + + # Create the destination directory if it + # doesn't already exist + unless (-d $dst_dir) { + # If necessary, allow a second check. Don't die just because the + # dir was created by another child (race condition): + mkpath($dst_dir) or (-d $dst_dir) + or die "Can't create directory $dst_dir\n"; + }; + $tmpfh = new File::Temp( + UNLINK => 1, + DIR => $dst_dir, + SUFFIX => '.tmp' + ); + $tmpfilename = $tmpfh->filename; + } + msg_info( $pretendString . "Transcoding \"$source\"" ); - my $convert_command = "\"$flaccmd\" @flacargs \"$source\"" . "| \"$lamecmd\" @lameargs - \"$tmpfilename\""; + my $convert_command = "\"$flaccmd\" @flacargs \"$source\"" . "| \"$lamecmd\" @lameargs - \"$tmpfilename\""; - $Options{debug} && msg("transcode: $convert_command"); + $Options{debug} && msg("transcode: $convert_command"); - # Convert the file (unless we're pretending} - my $exit_value; - if ( !$Options{pretend} ) { - $exit_value = system($convert_command); - } - else { - $exit_value = 0; - } + # Convert the file (unless we're pretending} + my $exit_value; + if ( !$Options{pretend} ) { + $exit_value = system($convert_command); + } + else { + $exit_value = 0; + } - $Options{debug} - && msg("Exit value from convert command: $exit_value"); + $Options{debug} + && msg("Exit value from convert command: $exit_value"); - if ($exit_value) { - msg("$convert_command failed with exit code $exit_value"); + if ($exit_value) { + msg("$convert_command failed with exit code $exit_value"); - # delete the destfile if it exists - unlink $tmpfilename; + # delete the destfile if it exists + unlink $tmpfilename; - # should check exit status of this command + # should check exit status of this command - exit($exit_value); - } + exit($exit_value); + } - if ( !$Options{pretend} ) { + if ( !$Options{pretend} ) { - # If we get here, assume the conversion has succeeded - $tmpfh->unlink_on_destroy(0); - $tmpfh->close; - croak "Failed to rename '$tmpfilename' to '$target' $!" - unless rename( $tmpfilename, $target ); + # If we get here, assume the conversion has succeeded + $tmpfh->unlink_on_destroy(0); + $tmpfh->close; + croak "Failed to rename '$tmpfilename' to '$target' $!" + unless rename( $tmpfilename, $target ); - # the destfile now exists! - $pflags{exists} = 1; + # the destfile now exists! + $pflags{exists} = 1; + + # and the tags need writing + $pflags{tags} = 1; + } - # and the tags need writing - $pflags{tags} = 1; - } - } if ( $Options{debug} ) { msg("pf_exists: $pflags{exists}"); msg("pf_tags: $pflags{tags}"); msg( "\$Options{pretend}: " . ( $Options{pretend} ? 'set' : 'not set' ) ); - } + } %$pflags_ref = %pflags; # this is only to minimize changes } From 6beccc78ede9cfd26892b00a4df9103d7829cec2 Mon Sep 17 00:00:00 2001 From: Carl Asplund Date: Sun, 17 Mar 2013 23:10:46 +0100 Subject: [PATCH 15/15] buffer message output (step 2/5) In this second step towards progress reporting the messages from transcoding and tagging are not display immediately on screen, but saved to an array variable and displayed once the respective forked child exits. Without this step, there is no real control over the order by with the messages appear when using several parallel processes, and progress reporting would look a complete mess. A callback subroutine is used to output the messages. It is called via the method 'run_on_finish' from the module Parallel::Forkmanager. --- flac2mp3.pl | 65 ++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 50 insertions(+), 15 deletions(-) diff --git a/flac2mp3.pl b/flac2mp3.pl index 9a5a76b..533f2d5 100755 --- a/flac2mp3.pl +++ b/flac2mp3.pl @@ -234,14 +234,34 @@ # it in perfect sync with (i.e. a mirror of) the source directory. delete_excess_files_from_dest($source_root, $target_root) if ( $Options{delete} ) ; +my $pm = new Parallel::ForkManager($Options{processes}); +$pm->run_on_finish( sub { + # This callback code is run after each transcode and outputs messages generated + # by the children + # + # According to the Parallel::Forkmanager documentation, the structure of the + # input data @_ to this callback function is as follows: + # ($pid, $exit_code, $ident, $exit_signal, $core_dump, $data_structure_reference) + my $src = $_[2]; # this is "$ident" + my $messages = $_[5]; # this is "$data_structure_reference" + + # display information about the transcoded/tagged file + foreach my $string ( @$messages ) { + msg_info($string); + } +}); # use parallel processing to launch multiple transcoding processes msg_info("Using $Options{processes} transcoding processes.\n"); -my $pm = new Parallel::ForkManager($Options{processes}); + foreach my $src_file (@flac_files) { - $pm->start and next; # Forks and returns the pid for the child - path_and_conversion($src_file); - $pm->finish; # Terminates the child process + $pm->start($src_file) and next; # forks here + + # process file and generate messages with file info + my $messageref = path_and_conversion($src_file); + + # terminate child process, send messages to callback sub + $pm->finish(0, $messageref ); } $pm->wait_all_children; @@ -447,6 +467,7 @@ sub get_md5_of_non_flac_file { sub path_and_conversion{ my $source = shift; my $target = $flac_mp3_files{$source}; + my @messages = (); $Options{debug} && msg("source: '$source'"); $Options{debug} && msg("target: '$target'"); @@ -458,18 +479,22 @@ sub path_and_conversion{ my $tags_to_update = preprocess_flac_tags( $source_tags ); # Step 3: Initialise file processing flags - my ($pflags) = examine_destfile_tags( $target, $tags_to_update ); - + my ($pflags, $mess) = examine_destfile_tags( $target, $tags_to_update ); + push @messages, @$mess; if ( ( !$$pflags{exists} || $$pflags{md5} || $Options{force} ) && !$Options{tagsonly} ) { # Step 4: Transcode the file based on the processing flags - transcode_file( $source, $target, $pflags ); + $mess = transcode_file( $source, $target, $pflags ); + push @messages, @$mess; }; # Step 5: Write the tags based on the processing flags - write_tags( $target, $tags_to_update, $pflags ); + $mess = write_tags( $target, $tags_to_update, $pflags ); + push @messages, @$mess; + + return \@messages; }; sub showusage { @@ -594,6 +619,7 @@ sub examine_destfile_tags { my $destfilename = shift; my $frames_ref = shift; my %frames_to_update = %$frames_ref; # this is only to minimize changes + my @return_messages = (); # Initialise file processing flags my %pflags = ( @@ -703,9 +729,11 @@ sub examine_destfile_tags { if ( $dest_text ne $srcframe ) { $pflags{tags} = 1; if ( $Options{tagdiff} ) { - msg("frame: '$frame'"); - msg("srcframe value: '$srcframe'"); - msg("destframe value: '$dest_text'"); + push @return_messages, ( + "frame: '$frame'", + "srcframe value: '$srcframe'", + "destframe value: '$dest_text'" + ); } } } @@ -728,7 +756,7 @@ sub examine_destfile_tags { msg( Dumper \%frames_to_update ); } - return \%pflags; + return \%pflags, \@return_messages; } sub transcode_file { @@ -736,6 +764,7 @@ sub transcode_file { my $target = shift; my $pflags_ref = shift; my %pflags = %$pflags_ref; # this is only to minimize changes + my @return_messages = (); # Transcode to a temp file in the destdir. # Rename the file if the conversion completes sucessfully @@ -765,7 +794,8 @@ sub transcode_file { ); $tmpfilename = $tmpfh->filename; } - msg_info( $pretendString . "Transcoding \"$source\"" ); + # Save message to be displayed on screen to the buffer + push @return_messages, $pretendString . "Transcoding \"$source\"" ; my $convert_command = "\"$flaccmd\" @flacargs \"$source\"" . "| \"$lamecmd\" @lameargs - \"$tmpfilename\""; @@ -817,6 +847,8 @@ sub transcode_file { } %$pflags_ref = %pflags; # this is only to minimize changes + + return \@return_messages; } sub write_tags { @@ -825,11 +857,13 @@ sub write_tags { my $pflags_ref = shift; my %frames_to_update = %$frames_ref; # this is only to minimize changes my %pflags = %$pflags_ref; # this is only to minimize changes + my @return_messages = (); # Write the tags if ( $pflags{exists} && ( $pflags{tags} || $Options{force} ) ) { - msg_info( $pretendString . "Writing tags to \"$destfilename\"" ); + # save message to be displayed on screen + push @return_messages, $pretendString . "Writing tags to \"$destfilename\""; if ( !$Options{pretend} ) { my $mp3 = MP3::Tag->new($destfilename); @@ -892,6 +926,7 @@ sub write_tags { # utime $srcstat->mtime, $srcstat->mtime, $destfilename; } } + return \@return_messages; } sub INT_Handler { @@ -967,4 +1002,4 @@ sub picsToAPICframes { # vim:set softtabstop=4: # vim:set shiftwidth=4: -__END__ \ No newline at end of file +__END__