       1 #!/usr/bin/perl -w
       2 #
       3 # File: JoinTextFiles.pl
       4 # Author: Manish Sud <msud@san.rr.com>
       5 #
       6 # Copyright (C) 2018 Manish Sud. All rights reserved.
       7 #
       8 # This file is part of MayaChemTools.
       9 #
      10 # MayaChemTools is free software; you can redistribute it and/or modify it under
      11 # the terms of the GNU Lesser General Public License as published by the Free
      12 # Software Foundation; either version 3 of the License, or (at your option) any
      13 # later version.
      14 #
      15 # MayaChemTools is distributed in the hope that it will be useful, but without
      16 # any warranty; without even the implied warranty of merchantability of fitness
      17 # for a particular purpose.  See the GNU Lesser General Public License for more
      18 # details.
      19 #
      20 # You should have received a copy of the GNU Lesser General Public License
      21 # along with MayaChemTools; if not, see <http://www.gnu.org/licenses/> or
      22 # write to the Free Software Foundation Inc., 59 Temple Place, Suite 330,
      23 # Boston, MA, 02111-1307, USA.
      24 #
      25
      26 use strict;
      27 use FindBin; use lib "$FindBin::Bin/../lib";
      28 use Getopt::Long;
      29 use File::Basename;
      30 use Text::ParseWords;
      31 use Benchmark;
      32 use FileUtil;
      33 use TextUtil;
      34
      35 my($ScriptName, %Options, $StartTime, $EndTime, $TotalTime);
      36
      37 # Autoflush STDOUT
      38 $| = 1;
      39
      40 # Starting message...
      41 $ScriptName = basename $0;
      42 print "\n$ScriptName:Starting...\n\n";
      43 $StartTime = new Benchmark;
      44
      45 # Get the options and setup script...
      46 SetupScriptUsage();
      47 if ($Options{help} || @ARGV < 1) {
      48   die GetUsageFromPod("$FindBin::Bin/$ScriptName");
      49 }
      50
      51 my(@TextFilesList);
      52 @TextFilesList = ExpandFileNames(\@ARGV, "csv tsv");
      53
      54 if (@TextFilesList == 1) {
      55   die "Error: Specify more than one Text file.\n";
      56 }
      57
      58 # Process options...
      59 print "Processing options...\n";
      60 my(%OptionsInfo);
      61 ProcessOptions();
      62
      63 # Setup information about input files...
      64 print "Checking input text files...\n";
      65 my(%TextFilesInfo);
      66 RetrieveTextFilesInfo();
      67
      68 # Join files...
      69 print "\nGenerating new text file $OptionsInfo{NewTextFile}...\n";
      70 JoinTextFiles();
      71
      72 print "\n$ScriptName:Done...\n\n";
      73
      74 $EndTime = new Benchmark;
      75 $TotalTime = timediff ($EndTime, $StartTime);
      76 print "Total time: ", timestr($TotalTime), "\n";
      77
      78 ###############################################################################
      79
      80 # Join all valid Text files...
      81 sub JoinTextFiles {
      82   my($FileIndex, $TextFile, $NewTextFile, $Line, $FirstColLabelsLine, $OutDelim, $OutQuote, $InDelim, @Words, @ColLabels);
      83
      84   $NewTextFile = $OptionsInfo{NewTextFile};
      85
      86   $FirstColLabelsLine = '';
      87
      88   $OutDelim = $OptionsInfo{OutDelim}; $OutQuote = $OptionsInfo{OutQuote};
      89
      90   open NEWTEXTFILE, ">$NewTextFile" or die "Error: Couldn't open $NewTextFile: $! \n";
      91   FILELIST: for $FileIndex (0 .. $#TextFilesList) {
      92     if (!$TextFilesInfo{FileOkay}[$FileIndex]) {
      93       next FILELIST;
      94     }
      95
      96     $TextFile = $TextFilesList[$FileIndex];
      97     $InDelim = $TextFilesInfo{InDelim}[$FileIndex];
      98
      99     print "\nProcessing file $TextFile...\n";
     100
     101     open TEXTFILE, "$TextFile" or die "Error: Couldn't open $TextFile: $! \n";
     102
     103     if ($OptionsInfo{Label}) {
     104       if ($OptionsInfo{Fast}) {
     105         $Line = GetTextLine(\*TEXTFILE);
     106         if (!$FirstColLabelsLine) {
     107           $FirstColLabelsLine = $Line;
     108           print NEWTEXTFILE "$FirstColLabelsLine\n";
     109         }
     110       }
     111       else {
     112         $Line = GetTextLine(\*TEXTFILE);
     113         if (!$FirstColLabelsLine) {
     114           @ColLabels = quotewords($InDelim, 0, $Line);
     115           $FirstColLabelsLine = JoinWords(\@ColLabels, $OutDelim, $OutQuote);
     116           print NEWTEXTFILE "$FirstColLabelsLine\n";
     117         }
     118       }
     119     }
     120
     121     while ($Line = GetTextLine(\*TEXTFILE)) {
     122       if (!$OptionsInfo{Fast}) {
     123         @Words = quotewords($InDelim, 0, $Line);
     124         $Line = JoinWords(\@Words, $OutDelim, $OutQuote);
     125       }
     126       print NEWTEXTFILE "$Line\n";
     127     }
     128     close TEXTFILE;
     129   }
     130
     131   close NEWTEXTFILE;
     132 }
     133
     134 # Retrieve information about Text files...
     135 sub RetrieveTextFilesInfo {
     136   my($Index, $TextFile, $InDelim, $Line, $FirstColLabelsLine, $ColLabelsLine, $FileDir, $FileName, $FileExt, @FirstColLabels, @ColLabels);
     137
     138   %TextFilesInfo = ();
     139   @{$TextFilesInfo{FileOkay}} = ();
     140   @{$TextFilesInfo{InDelim}} = ();
     141
     142   $FirstColLabelsLine = ''; $ColLabelsLine = '';
     143   @FirstColLabels = (); @ColLabels = ();
     144
     145   FILELIST: for $Index (0 .. $#TextFilesList) {
     146     $TextFilesInfo{FileOkay}[$Index] = 0;
     147     $TextFilesInfo{InDelim}[$Index] = "";
     148
     149     $TextFile = $TextFilesList[$Index];
     150     if (!(-e $TextFile)) {
     151       warn "Warning: Ignoring file $TextFile: It doesn't exist\n";
     152       next FILELIST;
     153     }
     154     if (!CheckFileType($TextFile, "csv tsv")) {
     155       warn "Warning: Ignoring file $TextFile: It's not a Text file\n";
     156       next FILELIST;
     157     }
     158
     159     ($FileDir, $FileName, $FileExt) = ParseFileName($TextFile);
     160     if ($FileExt =~ /^tsv$/i) {
     161       $InDelim = "\t";
     162     }
     163     else {
     164       $InDelim = "\,";
     165       if ($OptionsInfo{InDelim} !~ /^(comma|semicolon)$/i) {
     166         warn "Warning: Ignoring file $TextFile: The value specified, $OptionsInfo{InDelim}, for option \"--indelim\" is not valid for csv files\n";
     167         next FILELIST;
     168       }
     169       if ($OptionsInfo{InDelim} =~ /^semicolon$/i) {
     170         $InDelim = "\;";
     171       }
     172     }
     173
     174     if (! open TEXTFILE, "$TextFile") {
     175       warn "Warning: Ignoring file $TextFile: Couldn't open it: $! \n";
     176       next FILELIST;
     177     }
     178     $Line = GetTextLine(\*TEXTFILE);
     179     close TEXTFILE;
     180
     181     if ($OptionsInfo{Label}) {
     182       if (!$OptionsInfo{Fast}) {
     183         @ColLabels = quotewords($InDelim, 0, $Line);
     184         if ($FirstColLabelsLine) {
     185           if (@ColLabels != @FirstColLabels) {
     186             warn "Warning: Ignoring file $TextFile: The number of columns in this file, ", scalar(@ColLabels), ", is different from the number of columns, ", scalar(@FirstColLabels), ", in the first valid text file. \n";
     187             next FILELIST;
     188           }
     189           $ColLabelsLine = JoinWords(\@ColLabels, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
     190           if ($ColLabelsLine ne $FirstColLabelsLine) {
     191             warn "Warning: Ignoring file $TextFile: The column names in this file are different from those in first valid text file.\nColumnlabels in first valid text file: $FirstColLabelsLine \nColumnlabels in current text file: $ColLabelsLine\n";
     192             next FILELIST;
     193           }
     194         }
     195         else {
     196           @FirstColLabels = @ColLabels;
     197           $FirstColLabelsLine = JoinWords(\@FirstColLabels, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
     198         }
     199       }
     200     }
     201
     202     $TextFilesInfo{FileOkay}[$Index] = 1;
     203     $TextFilesInfo{InDelim}[$Index] = $InDelim;
     204   }
     205 }
     206
     207 # Process option values...
     208 sub ProcessOptions {
     209   my($FileDir, $FileName, $FileExt, $NewTextFile, $Index);
     210
     211   %OptionsInfo = ();
     212
     213   $OptionsInfo{Fast} = $Options{fast} ? $Options{fast} : 0;
     214
     215   $OptionsInfo{InDelim} = $Options{indelim};
     216   $OptionsInfo{Label} = ($Options{label} =~ /^yes$/i) ? 1 : 0;
     217
     218   $OptionsInfo{OutFileRoot} = $Options{root} ? $Options{root} : undef;
     219   $OptionsInfo{Overwrite} = $Options{overwrite} ? $Options{overwrite} : undef;
     220
     221   $OptionsInfo{OutDelim} = ($Options{outdelim} =~ /^tab$/i ) ? "\t" : (($Options{outdelim} =~ /^semicolon$/i) ? "\;" : "\,");
     222   $OptionsInfo{OutQuote} = ($Options{quote} =~ /^yes$/i) ? 1 : 0;
     223
     224   if ($Options{root}) {
     225     $FileDir = ""; $FileName = ""; $FileExt = "";
     226     ($FileDir, $FileName, $FileExt) = ParseFileName($Options{root});
     227     if ($FileName && $FileExt) {
     228       $NewTextFile = $FileName;
     229     }
     230     else {
     231       $NewTextFile =  $Options{root};
     232     }
     233   }
     234   else {
     235     $FileDir = ""; $FileName = ""; $FileExt = "";
     236     ($FileDir, $FileName, $FileExt) = ParseFileName($TextFilesList[0]);
     237     $NewTextFile = $FileName . "1To" . @TextFilesList . "Joined";
     238   }
     239
     240   if ($Options{outdelim} =~ /^tab$/i) {
     241     $NewTextFile .= ".tsv";
     242   }
     243   else {
     244     $NewTextFile .= ".csv";
     245   }
     246
     247   if (!$Options{overwrite}) {
     248     if (-e $NewTextFile) {
     249       die "Error: The file $NewTextFile already exists.\n";
     250     }
     251   }
     252
     253   if ($Options{root}) {
     254     for $Index (0 .. $#TextFilesList) {
     255       if (lc($NewTextFile) eq lc($TextFilesList[$Index])) {
     256         die "Error: Output filename, $NewTextFile, is similar to a input file name.\nSpecify a different name using \"-r --root\" option or use default name.\n";
     257       }
     258     }
     259   }
     260
     261   $OptionsInfo{NewTextFile} = $NewTextFile;
     262 }
     263
     264 # Setup script usage  and retrieve command line arguments specified using various options...
     265 sub SetupScriptUsage {
     266
     267   # Retrieve all the options...
     268   %Options = ();
     269   $Options{label} = "yes";
     270   $Options{indelim} = "comma";
     271   $Options{outdelim} = "comma";
     272   $Options{quote} = "yes";
     273   if (!GetOptions(\%Options, "fast|f", "help|h", "indelim=s", "label|l=s", "outdelim=s", "overwrite|o", "quote|q=s", "root|r=s", "workingdir|w=s")) {
     274     die "\nTo get a list of valid options and their values, use \"$ScriptName -h\" or\n\"perl -S $ScriptName -h\" command and try again...\n";
     275   }
     276   if ($Options{workingdir}) {
     277     if (! -d $Options{workingdir}) {
     278       die "Error: The value specified, $Options{workingdir},  for option \"-w --workingdir\" is not a directory name.\n";
     279     }
     280     chdir $Options{workingdir} or die "Error: Couldn't chdir $Options{workingdir}: $! \n";
     281   }
     282   if ($Options{indelim} !~ /^(comma|semicolon)$/i) {
     283     die "Error: The value specified, $Options{indelim}, for option \"--indelim\" is not valid. Allowed values: comma or semicolon\n";
     284   }
     285   if ($Options{outdelim} !~ /^(comma|semicolon|tab)$/i) {
     286     die "Error: The value specified, $Options{outdelim}, for option \"--outdelim\" is not valid. Allowed values: comma, tab, or semicolon\n";
     287   }
     288   if ($Options{quote} !~ /^(yes|no)$/i) {
     289     die "Error: The value specified, $Options{quote}, for option \"-q --quote\" is not valid. Allowed values: yes or no\n";
     290   }
     291   if ($Options{label} !~ /^(yes|no)$/i) {
     292     die "Error: The value specified, $Options{label}, for option \"-l --label\" is not valid. Allowed values: yes or no\n";
     293   }
     294 }
     295