       1 #!/usr/bin/perl -w
       2 #
       3 # File: SortTextFiles.pl
       4 # Author: Manish Sud <msud@san.rr.com>
       5 #
       6 # Copyright (C) 2018 Manish Sud. All rights reserved.
       7 #
       8 # This file is part of MayaChemTools.
       9 #
      10 # MayaChemTools is free software; you can redistribute it and/or modify it under
      11 # the terms of the GNU Lesser General Public License as published by the Free
      12 # Software Foundation; either version 3 of the License, or (at your option) any
      13 # later version.
      14 #
      15 # MayaChemTools is distributed in the hope that it will be useful, but without
      16 # any warranty; without even the implied warranty of merchantability of fitness
      17 # for a particular purpose.  See the GNU Lesser General Public License for more
      18 # details.
      19 #
      20 # You should have received a copy of the GNU Lesser General Public License
      21 # along with MayaChemTools; if not, see <http://www.gnu.org/licenses/> or
      22 # write to the Free Software Foundation Inc., 59 Temple Place, Suite 330,
      23 # Boston, MA, 02111-1307, USA.
      24 #
      25
      26 use strict;
      27 use FindBin; use lib "$FindBin::Bin/../lib";
      28 use Getopt::Long;
      29 use File::Basename;
      30 use Text::ParseWords;
      31 use Benchmark;
      32 use FileUtil;
      33 use TextUtil;
      34
      35 my($ScriptName, %Options, $StartTime, $EndTime, $TotalTime);
      36
      37 # Autoflush STDOUT
      38 $| = 1;
      39
      40 # Starting message...
      41 $ScriptName = basename($0);
      42 print "\n$ScriptName: Starting...\n\n";
      43 $StartTime = new Benchmark;
      44
      45 # Get the options and setup script...
      46 SetupScriptUsage();
      47 if ($Options{help} || @ARGV < 1) {
      48   die GetUsageFromPod("$FindBin::Bin/$ScriptName");
      49 }
      50
      51 my(@TextFilesList);
      52 @TextFilesList = ExpandFileNames(\@ARGV, "csv tsv");
      53
      54 print "Processing options...\n";
      55 my(%OptionsInfo);
      56 ProcessOptions();
      57
      58 print "Checking input text file(s)...\n";
      59 my(%TextFilesInfo);
      60 RetrieveTextFilesInfo();
      61 ProcessColumnsInfo();
      62
      63 # Generate output files...
      64 my($FileIndex);
      65 if (@TextFilesList > 1) {
      66   print "\nProcessing text files...\n";
      67 }
      68 for $FileIndex (0 .. $#TextFilesList) {
      69   if ($TextFilesInfo{FileOkay}[$FileIndex]) {
      70     print "\nProcessing file $TextFilesList[$FileIndex]...\n";
      71     SortTextFile($FileIndex);
      72   }
      73 }
      74 print "\n$ScriptName:Done...\n\n";
      75
      76 $EndTime = new Benchmark;
      77 $TotalTime = timediff ($EndTime, $StartTime);
      78 print "Total time: ", timestr($TotalTime), "\n";
      79
      80 ###############################################################################
      81
      82 # Sort it out...
      83 sub SortTextFile {
      84   my($Index) = @_;
      85   my($TextFile, $NewTextFile, $KeyCol, $Line, $KeyColValue, $InDelim, @ColLabels, @LineWords);
      86
      87   $TextFile = $TextFilesList[$Index];
      88   $InDelim = $TextFilesInfo{InDelim}[$Index];
      89   $NewTextFile = $TextFilesInfo{OutFile}[$Index];
      90   $KeyCol = $TextFilesInfo{KeyColNum}[$Index];
      91   @ColLabels = @{$TextFilesInfo{ColLabels}[$Index]};
      92
      93   print "Generating new Text file $NewTextFile...\n";
      94   open NEWTEXTFILE, ">$NewTextFile" or die "Error: Couldn't open $NewTextFile: $! \n";
      95   open TEXTFILE, "$TextFile" or die "Error: Can't open $TextFile: $! \n";
      96
      97   # Skip over column labels from old file...
      98   $Line = GetTextLine(\*TEXTFILE);
      99
     100   # Add column lablels in new file...
     101   $Line = JoinWords(\@ColLabels, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
     102   print NEWTEXTFILE "$Line\n";
     103
     104   # Go over all rows and store the lines using key value as hash...
     105   my(%KeyToLinesMap, @InvalidDataLines, $LineCount);
     106
     107   %KeyToLinesMap = ();
     108   @InvalidDataLines = ();
     109   $LineCount = 1;
     110   TEXTLINE: while ($Line = GetTextLine(\*TEXTFILE)) {
     111     @LineWords = quotewords($InDelim, 0, $Line);
     112     $LineCount++;
     113     if ($KeyCol < @LineWords) {
     114       $KeyColValue = $LineWords[$KeyCol];
     115       if (!IsNotEmpty($KeyColValue)) {
     116         $Line = JoinWords(\@LineWords, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
     117         push @InvalidDataLines, $Line;
     118         if ($OptionsInfo{DetailLevel} >= 3 ) {
     119           print "Ignoring line $LineCount: Contains empty value for key column $ColLabels[$KeyCol]: $Line\n";
     120         }
     121         elsif ($OptionsInfo{DetailLevel} >= 2) {
     122           print "Ignoring line $LineCount: Contains empty value for key column $ColLabels[$KeyCol]...\n";
     123         }
     124         next TEXTLINE;
     125       }
     126       if ($OptionsInfo{KeyData} =~ /^numeric$/i) {
     127         if (!IsFloat($KeyColValue)) {
     128           $Line = JoinWords(\@LineWords, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
     129           push @InvalidDataLines, $Line;
     130           if ($OptionsInfo{DetailLevel} >= 3 ) {
     131             print "Line number $LineCount: Contains non-numerical value for key column $ColLabels[$KeyCol]: $Line\n";
     132           }
     133           elsif ($OptionsInfo{DetailLevel} >= 2) {
     134             print "Line number $LineCount: Contains non-numerical value for key column $ColLabels[$KeyCol]...\n";
     135           }
     136           next TEXTLINE;
     137         }
     138       }
     139       if (exists($KeyToLinesMap{$KeyColValue})) {
     140         # Append to existing line...
     141         $Line = JoinWords(\@LineWords, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
     142         $KeyToLinesMap{$KeyColValue} .= "\n" . $Line;
     143       }
     144       else {
     145         $Line = JoinWords(\@LineWords, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
     146         $KeyToLinesMap{$KeyColValue} = $Line;
     147       }
     148     }
     149   }
     150   if ($OptionsInfo{Sort} =~ /^ascending$/i) {
     151     if ($OptionsInfo{KeyData} =~ /^alphanumeric$/i) {
     152       for $KeyColValue (sort { lc($a) cmp lc($b) } keys %KeyToLinesMap ) {
     153         print NEWTEXTFILE "$KeyToLinesMap{$KeyColValue}\n";
     154       }
     155     }
     156     else {
     157       for $KeyColValue (sort { $a <=> $b } keys %KeyToLinesMap ) {
     158         print NEWTEXTFILE "$KeyToLinesMap{$KeyColValue}\n";
     159       }
     160     }
     161   }
     162   else {
     163     if ($OptionsInfo{KeyData} =~ /^alphanumeric$/i) {
     164       for $KeyColValue (sort { lc($b) cmp lc($a) } keys %KeyToLinesMap ) {
     165         print NEWTEXTFILE "$KeyToLinesMap{$KeyColValue}\n";
     166       }
     167     }
     168     else {
     169       for $KeyColValue (sort { $b <=> $a } keys %KeyToLinesMap ) {
     170         print NEWTEXTFILE "$KeyToLinesMap{$KeyColValue}\n";
     171       }
     172     }
     173   }
     174   # Write out the lines with invalid data...
     175   if (@InvalidDataLines) {
     176     print "Placing ", scalar(@InvalidDataLines)," line(s) with invalid column key data at the end...\n";
     177     for $Line (@InvalidDataLines) {
     178       print NEWTEXTFILE "$Line\n";
     179     }
     180   }
     181   close NEWTEXTFILE;
     182   close TEXTFILE;
     183
     184 }
     185
     186 # Retrieve information about input text files...
     187 sub RetrieveTextFilesInfo {
     188   my($Index, $TextFile, $FileDir, $FileName, $FileExt, $InDelim, $Line, @ColLabels, $OutFileRoot,  $OutFile, $ColNum, $ColLabel);
     189
     190   %TextFilesInfo = ();
     191
     192   @{$TextFilesInfo{FileOkay}} = ();
     193   @{$TextFilesInfo{ColCount}} = ();
     194   @{$TextFilesInfo{ColLabels}} = ();
     195   @{$TextFilesInfo{ColLabelToNumMap}} = ();
     196   @{$TextFilesInfo{InDelim}} = ();
     197   @{$TextFilesInfo{OutFile}} = ();
     198
     199   FILELIST: for $Index (0 .. $#TextFilesList) {
     200     $TextFile = $TextFilesList[$Index];
     201
     202     $TextFilesInfo{FileOkay}[$Index] = 0;
     203     $TextFilesInfo{ColCount}[$Index] = 0;
     204     $TextFilesInfo{InDelim}[$Index] = "";
     205     $TextFilesInfo{OutFile}[$Index] = "";
     206     @{$TextFilesInfo{ColLabels}[$Index]} = ();
     207     %{$TextFilesInfo{ColLabelToNumMap}[$Index]} = ();
     208
     209     if (!(-e $TextFile)) {
     210       warn "Warning: Ignoring file $TextFile: It doesn't exist\n";
     211       next FILELIST;
     212     }
     213     if (!CheckFileType($TextFile, "csv tsv")) {
     214       warn "Warning: Ignoring file $TextFile: It's not a csv or tsv file\n";
     215       next FILELIST;
     216     }
     217     ($FileDir, $FileName, $FileExt) = ParseFileName($TextFile);
     218     if ($FileExt =~ /^tsv$/i) {
     219       $InDelim = "\t";
     220     }
     221     else {
     222       $InDelim = "\,";
     223       if ($Options{indelim} !~ /^(comma|semicolon)$/i) {
     224         warn "Warning: Ignoring file $TextFile: The value specified, $Options{indelim}, for option \"--indelim\" is not valid for csv files\n";
     225         next FILELIST;
     226       }
     227       if ($Options{indelim} =~ /^semicolon$/i) {
     228         $InDelim = "\;";
     229       }
     230     }
     231
     232     if (!open TEXTFILE, "$TextFile") {
     233       warn "Warning: Ignoring file $TextFile: Couldn't open it: $! \n";
     234       next FILELIST;
     235     }
     236
     237     $Line = GetTextLine(\*TEXTFILE);
     238     @ColLabels = quotewords($InDelim, 0, $Line);
     239     close TEXTFILE;
     240
     241     $FileDir = ""; $FileName = ""; $FileExt = "";
     242     ($FileDir, $FileName, $FileExt) = ParseFileName($TextFile);
     243     $FileExt = "csv";
     244     if ($Options{outdelim} =~ /^tab$/i) {
     245       $FileExt = "tsv";
     246     }
     247     if ($Options{root} && (@TextFilesList == 1)) {
     248       my ($RootFileDir, $RootFileName, $RootFileExt) = ParseFileName($Options{root});
     249       if ($RootFileName && $RootFileExt) {
     250         $FileName = $RootFileName;
     251       }
     252       else {
     253         $FileName = $Options{root};
     254       }
     255       $OutFileRoot = $FileName;
     256     }
     257     else {
     258       $OutFileRoot = $FileName . "SortedByColumn";
     259     }
     260
     261     $OutFile = $OutFileRoot . ".$FileExt";
     262     if (lc($OutFile) eq lc($TextFile)) {
     263       warn "Warning: Ignoring file $TextFile:Output file name, $OutFile, is same as input text file name, $TextFile\n";
     264       next FILELIST;
     265     }
     266     if (!$Options{overwrite}) {
     267       if (-e $OutFile) {
     268         warn "Warning: Ignoring file $TextFile: The file $OutFile already exists\n";
     269         next FILELIST;
     270       }
     271     }
     272
     273     $TextFilesInfo{FileOkay}[$Index] = 1;
     274     $TextFilesInfo{InDelim}[$Index] = $InDelim;
     275     $TextFilesInfo{OutFile}[$Index] = "$OutFile";
     276
     277     $TextFilesInfo{ColCount}[$Index] = @ColLabels;
     278     push @{$TextFilesInfo{ColLabels}[$Index]}, @ColLabels;
     279     for $ColNum (0 .. $#ColLabels) {
     280       $ColLabel = $ColLabels[$ColNum];
     281       $TextFilesInfo{ColLabelToNumMap}[$Index]{$ColLabel} = $ColNum;
     282     }
     283   }
     284
     285 }
     286
     287 # Make sure specified key column are okay...
     288 sub ProcessColumnsInfo {
     289   my($Index, $TextFile, $SpecifiedKeyCol);
     290
     291   @{$TextFilesInfo{KeyColNum}} = ();
     292
     293   $SpecifiedKeyCol = $OptionsInfo{SpecifiedKeyCol};
     294
     295   FILELIST: for $Index (0 .. $#TextFilesList) {
     296     $TextFile = $TextFilesList[$Index];
     297     $TextFilesInfo{KeyColNum}[$Index] = 0;
     298
     299     if ($TextFilesInfo{FileOkay}[$Index]) {
     300       my($KeyColNum, $KeyColValid);
     301
     302       $KeyColNum = 0;
     303       $KeyColValid = 1;
     304       if ($SpecifiedKeyCol) {
     305         if ($OptionsInfo{Mode} =~ /^colnum$/i) {
     306           if ($SpecifiedKeyCol <= $TextFilesInfo{ColCount}[$Index]) {
     307             $KeyColNum = $SpecifiedKeyCol - 1;
     308           }
     309           else {
     310             $KeyColValid = 0;
     311           }
     312         }
     313         else {
     314           if (exists($TextFilesInfo{ColLabelToNumMap}[$Index]{$SpecifiedKeyCol})) {
     315             $KeyColNum =  $TextFilesInfo{ColLabelToNumMap}[$Index]{$SpecifiedKeyCol};
     316           }
     317           else {
     318             $KeyColValid = 0;
     319           }
     320         }
     321       }
     322       if ($KeyColValid) {
     323         $TextFilesInfo{KeyColNum}[$Index] = $KeyColNum;
     324       }
     325       else {
     326         warn "Warning: Ignoring file $TextFile: Column key specified, $SpecifiedKeyCol, using \"k --key\" option doesn't exist\n";
     327         $TextFilesInfo{FileOkay}[$Index] = 0;
     328       }
     329     }
     330   }
     331 }
     332
     333 # Process option values...
     334 sub ProcessOptions {
     335   %OptionsInfo = ();
     336
     337   $OptionsInfo{Mode} = $Options{mode};
     338
     339   $OptionsInfo{DetailLevel} = $Options{detail};
     340
     341   $OptionsInfo{Sort} = $Options{sort};
     342   $OptionsInfo{KeyData} = $Options{keydata};
     343
     344   $OptionsInfo{InDelim} = $Options{indelim};
     345
     346   $OptionsInfo{OutDelim} = ($Options{outdelim} =~ /^tab$/i ) ? "\t" : (($Options{outdelim} =~ /^semicolon$/i) ? "\;" : "\,");
     347   $OptionsInfo{OutQuote} = ($Options{quote} =~ /^yes$/i) ? 1 : 0;
     348
     349   $OptionsInfo{Overwrite} = defined $Options{overwrite} ? $Options{overwrite} : undef;
     350   $OptionsInfo{Root} = defined $Options{root} ? $Options{root} : undef;
     351
     352   $OptionsInfo{Key} = defined $Options{key} ? $Options{key} : undef;
     353   $OptionsInfo{SpecifiedKeyCol} = "";
     354
     355   if (defined $Options{key}) {
     356     $OptionsInfo{SpecifiedKeyCol} = $Options{key};
     357     if ($Options{mode} =~ /^colnum$/i) {
     358       if (!IsPositiveInteger($OptionsInfo{SpecifiedKeyCol})) {
     359         die "Error: Invalid value $Options{key} specified using \"-k --key\" option: Allowed values: > 0\n";
     360       }
     361     }
     362   }
     363 }
     364
     365 # Setup script usage  and retrieve command line arguments specified using various options...
     366 sub SetupScriptUsage {
     367   %Options = ();
     368
     369   $Options{detail} = 1;
     370   $Options{mode} = "colnum";
     371   $Options{sort} = "ascending";
     372   $Options{keydata} = "numeric";
     373   $Options{indelim} = "comma";
     374   $Options{outdelim} = "comma";
     375   $Options{quote} = "yes";
     376   if (!GetOptions(\%Options, "detail|d=i", "help|h", "indelim=s", "key|k=s", "keydata=s", "mode|m=s", "outdelim=s", "overwrite|o", "quote|q=s", "root|r=s", "sort|s=s", "workingdir|w=s")) {
     377     die "\nTo get a list of valid options and their values, use \"$ScriptName -h\" or\n\"perl -S $ScriptName -h\" command and try again...\n";
     378   }
     379   if ($Options{workingdir}) {
     380     if (! -d $Options{workingdir}) {
     381       die "Error: The value specified, $Options{workingdir}, for option \"-w --workingdir\" is not a directory name.\n";
     382     }
     383     chdir $Options{workingdir} or die "Error: Couldn't chdir $Options{workingdir}: $! \n";
     384   }
     385   if ($Options{mode} !~ /^(colnum|collabel)$/i) {
     386     die "Error: The value specified, $Options{mode}, for option \"-m --mode\" is not valid. Allowed values: colnum or collabel\n";
     387   }
     388   if ($Options{keydata} !~ /^(numeric|alphanumeric)$/i) {
     389     die "Error: The value specified, $Options{keydata}, for option \"--keydata\" is not valid. Allowed values: numeric or alphanumeric\n";
     390   }
     391   if ($Options{indelim} !~ /^(comma|semicolon)$/i) {
     392     die "Error: The value specified, $Options{indelim}, for option \"--indelim\" is not valid. Allowed values: comma or semicolon\n";
     393   }
     394   if ($Options{outdelim} !~ /^(comma|semicolon|tab)$/i) {
     395     die "Error: The value specified, $Options{outdelim}, for option \"--outdelim\" is not valid. Allowed values: comma, tab, or semicolon\n";
     396   }
     397   if ($Options{quote} !~ /^(yes|no)$/i) {
     398     die "Error: The value specified, $Options{quote}, for option \"-q --quote\" is not valid. Allowed values: yes or no\n";
     399   }
     400   if ($Options{sort} !~ /^(ascending|descending)$/i) {
     401     die "Error: The value specified, $Options{sort}, for option \"-s --sort\" is not valid. Allowed values: ascending or descending\n";
     402   }
     403   if (!IsPositiveInteger($Options{detail})) {
     404     die "Error: The value specified, $Options{detail}, for option \"-d --detail\" is not valid. Allowed values: > 0\n";
     405   }
     406 }
     407