       1 #!/usr/bin/perl -w
       2 #
       3 # File: InfoTextFiles.pl
       4 # Author: Manish Sud <msud@san.rr.com>
       5 #
       6 # Copyright (C) 2018 Manish Sud. All rights reserved.
       7 #
       8 # This file is part of MayaChemTools.
       9 #
      10 # MayaChemTools is free software; you can redistribute it and/or modify it under
      11 # the terms of the GNU Lesser General Public License as published by the Free
      12 # Software Foundation; either version 3 of the License, or (at your option) any
      13 # later version.
      14 #
      15 # MayaChemTools is distributed in the hope that it will be useful, but without
      16 # any warranty; without even the implied warranty of merchantability of fitness
      17 # for a particular purpose.  See the GNU Lesser General Public License for more
      18 # details.
      19 #
      20 # You should have received a copy of the GNU Lesser General Public License
      21 # along with MayaChemTools; if not, see <http://www.gnu.org/licenses/> or
      22 # write to the Free Software Foundation Inc., 59 Temple Place, Suite 330,
      23 # Boston, MA, 02111-1307, USA.
      24 #
      25
      26 use strict;
      27 use FindBin; use lib "$FindBin::Bin/../lib";
      28 use Getopt::Long;
      29 use File::Basename;
      30 use Text::ParseWords;
      31 use Benchmark;
      32 use FileUtil;
      33 use TextUtil;
      34
      35 my($ScriptName, %Options, $StartTime, $EndTime, $TotalTime);
      36
      37 # Autoflush STDOUT
      38 $| = 1;
      39
      40 # Starting message...
      41 $ScriptName = basename($0);
      42 print "\n$ScriptName: Starting...\n\n";
      43 $StartTime = new Benchmark;
      44
      45 # Get the options and setup script...
      46 SetupScriptUsage();
      47 if ($Options{help} || @ARGV < 1) {
      48   die GetUsageFromPod("$FindBin::Bin/$ScriptName");
      49 }
      50
      51 my(@TextFilesList);
      52 @TextFilesList = ExpandFileNames(\@ARGV, "csv tsv");
      53
      54 # Process options...
      55 print "Processing options...\n";
      56 my(%OptionsInfo);
      57 ProcessOptions();
      58
      59 print "Checking input text file(s)...\n";
      60 my(%TextFilesInfo);
      61 RetrieveTextFilesInfo();
      62 ProcessColumnsInfo();
      63
      64 # Generate output files...
      65 my($FileIndex);
      66 if (@TextFilesList > 1) {
      67   print "\nProcessing text files...\n";
      68 }
      69 for $FileIndex (0 .. $#TextFilesList) {
      70   if ($TextFilesInfo{FileOkay}[$FileIndex]) {
      71     print "\nProcessing file $TextFilesList[$FileIndex]...\n";
      72     ListTextFileInfo($FileIndex);
      73   }
      74 }
      75 ListTotalSizeOfFiles();
      76
      77 print "\n$ScriptName:Done...\n\n";
      78
      79 $EndTime = new Benchmark;
      80 $TotalTime = timediff ($EndTime, $StartTime);
      81 print "Total time: ", timestr($TotalTime), "\n";
      82
      83 ###############################################################################
      84
      85 # List appropriate information...
      86 sub ListTextFileInfo {
      87   my($Index) = @_;
      88   my($TextFile,  $Line, $InDelim, $LineCount, $EmptyLinesCount, $EmptyColDataLinesCount, $GreaterThanMaxColLinesCount, $Label, $Value, $ColNum, $EmptyColValueFound, $PrintTextLine, $NonNumericalDataFound, @ColLabels, @LineWords, %EmptyColValuesCountMap, %NonEmptyColValuesCountMap, %SpecifiedNonNumericalColValuesCountMap, %NonNumericalColValuesCountMap, %NumericalColValuesCountMap,);
      89
      90   $TextFile = $TextFilesList[$Index];
      91   $InDelim = $TextFilesInfo{InDelim}[$Index];
      92   @ColLabels = @{$TextFilesInfo{ColLabels}[$Index]};
      93
      94   open TEXTFILE, "$TextFile" or die "Error: Can't open $TextFile: $! \n";
      95
      96   $LineCount = 0;
      97   $EmptyLinesCount = 0;
      98   $EmptyColDataLinesCount = 0;
      99   $GreaterThanMaxColLinesCount = 0;
     100
     101   %EmptyColValuesCountMap = ();
     102   %NonEmptyColValuesCountMap = ();
     103   %SpecifiedNonNumericalColValuesCountMap = ();
     104   %NonNumericalColValuesCountMap = ();
     105   %NumericalColValuesCountMap = ();
     106
     107   if ($OptionsInfo{ParseLines}) {
     108     # Skip over column labels from old file...
     109     if (<TEXTFILE>) {
     110       $LineCount++;
     111       LINE: while ($Line = <TEXTFILE>) {
     112         $LineCount++;
     113         $PrintTextLine = 0;
     114         $Line =~ s/(\r\n)|(\r)|\n//g;
     115         @LineWords = quotewords($InDelim, 0, $Line);
     116         if ($OptionsInfo{CountEmpty}) {
     117           # Count lines with no data...
     118           if (!@LineWords) {
     119             $EmptyLinesCount++;
     120             if ($OptionsInfo{DetailLevel} >= 2) {
     121               print "Line number $LineCount is empty...\n";
     122             }
     123             next LINE;
     124           }
     125           # Count lines with empty data for some columns...
     126           $EmptyColValueFound = 0;
     127           VALUE: for $Value (@LineWords) {
     128               if (!IsNotEmpty($Value)) {
     129                 $EmptyColValueFound = 1;
     130                 next VALUE;
     131               }
     132           }
     133           if ($EmptyColValueFound) {
     134             $EmptyColDataLinesCount++;
     135             if ($OptionsInfo{DetailLevel} >= 2) {
     136               print "Line number $LineCount contains empty column value(s)...\n";
     137             }
     138             $PrintTextLine = ($OptionsInfo{DetailLevel} >= 3) ? 1 : 0;
     139           }
     140           # Count lines with columns greater than the column label line...
     141           if (@LineWords > @ColLabels) {
     142             $GreaterThanMaxColLinesCount++;
     143             if ($OptionsInfo{DetailLevel} >= 2) {
     144               print "Line number $LineCount contains more than ", scalar(@ColLabels), " columns...\n";
     145             }
     146             $PrintTextLine = ($OptionsInfo{DetailLevel} >= 3) ? 1 : 0;
     147           }
     148           # Count empty values for each coulmn...
     149           for $ColNum (0 .. $#LineWords) {
     150             if ($ColNum < @ColLabels) {
     151               $Label = $ColLabels[$ColNum];
     152               if (IsNotEmpty($LineWords[$ColNum])) {
     153                 if (exists($NonEmptyColValuesCountMap{$Label})) {
     154                   $NonEmptyColValuesCountMap{$Label} += 1;
     155                 }
     156                 else {
     157                   $NonEmptyColValuesCountMap{$Label} = 1;
     158                 }
     159               }
     160               else {
     161                 $PrintTextLine = ($OptionsInfo{DetailLevel} >= 3) ? 1 : 0;
     162                 if (exists($EmptyColValuesCountMap{$Label})) {
     163                   $EmptyColValuesCountMap{$Label} += 1;
     164                 }
     165                 else {
     166                   $EmptyColValuesCountMap{$Label} = 1;
     167                 }
     168               }
     169             }
     170           }
     171         }
     172         if ($OptionsInfo{CheckData}) {
     173           for $ColNum (0 .. $#LineWords) {
     174             if ($ColNum < @ColLabels) {
     175               if (IsNumerical($LineWords[$ColNum])) {
     176                 $Label = $ColLabels[$ColNum];
     177                 if (exists($NumericalColValuesCountMap{$Label})) {
     178                   $NumericalColValuesCountMap{$Label} += 1;
     179                 }
     180                 else {
     181                   $NumericalColValuesCountMap{$Label} = 1;
     182                 }
     183               }
     184               else {
     185                 $Label = $ColLabels[$ColNum];
     186                 if (IsNotEmpty($LineWords[$ColNum])) {
     187                   if (exists($NonNumericalColValuesCountMap{$Label})) {
     188                     $NonNumericalColValuesCountMap{$Label} += 1;
     189                   }
     190                   else {
     191                     $NonNumericalColValuesCountMap{$Label} = 1;
     192                   }
     193                 }
     194               }
     195             }
     196           }
     197         }
     198         if ($OptionsInfo{CheckNumericalData}) {
     199           $NonNumericalDataFound = 0;
     200           for $ColNum (@{$TextFilesInfo{NumericalDataColNums}[$Index]}) {
     201             if ($ColNum < @LineWords) {
     202               if (!IsNumerical($LineWords[$ColNum])) {
     203                 $NonNumericalDataFound = 1;
     204                 $Label = $ColLabels[$ColNum];
     205                 if (exists($SpecifiedNonNumericalColValuesCountMap{$Label})) {
     206                   $SpecifiedNonNumericalColValuesCountMap{$Label} += 1;
     207                 }
     208                 else {
     209                   $SpecifiedNonNumericalColValuesCountMap{$Label} = 1;
     210                 }
     211               }
     212             }
     213           }
     214           if ($NonNumericalDataFound) {
     215             $PrintTextLine = ($OptionsInfo{DetailLevel} >= 3) ? 1 : 0;
     216             if ($OptionsInfo{DetailLevel} >=2 ) {
     217               print "Line number $LineCount contains non-numerical data for some specified column(s)...\n";
     218             }
     219           }
     220         }
     221         if ($PrintTextLine) {
     222           print "Line $LineCount: $Line\n\n";
     223         }
     224       }
     225     }
     226   }
     227   else {
     228     while (<TEXTFILE>) {
     229       $LineCount++;
     230     }
     231   }
     232   close TEXTFILE;
     233
     234   print "\nNumber of lines: $LineCount\n";
     235   print "Number of columns: $TextFilesInfo{ColCount}[$Index]\n";
     236   print "Column labels: ", JoinWords(\@ColLabels, ", ", 1), "\n";
     237
     238   if ($OptionsInfo{CountEmpty}) {
     239     print "\nNumber of lines with no data: $EmptyLinesCount\n";
     240     print "Number of lines with some missing column data: $EmptyColDataLinesCount\n";
     241     print "Number of lines containing greater than ", scalar(@ColLabels), " columns: $GreaterThanMaxColLinesCount\n";
     242     PrintDataInformation("Number of non-empty values for each column(s)", \@ColLabels, \%NonEmptyColValuesCountMap);
     243     PrintDataInformation("Number of empty values for each column(s)", \@ColLabels, \%EmptyColValuesCountMap);
     244   }
     245
     246   if ($OptionsInfo{CheckData}) {
     247     print "\n";
     248     PrintDataInformation("Number of non-numerical data values for each column(s)", \@ColLabels, \%NonNumericalColValuesCountMap);
     249     PrintDataInformation("Number of numerical data values for each column(s)", \@ColLabels, \%NumericalColValuesCountMap);
     250     print "\n";
     251   }
     252
     253   if ($OptionsInfo{CheckNumericalData} && @{$TextFilesInfo{NumericalDataColLabels}[$Index]}) {
     254     PrintDataInformation("Number of non-numerical data values for each column(s)", \@{$TextFilesInfo{NumericalDataColLabels}[$Index]}, \%SpecifiedNonNumericalColValuesCountMap);
     255   }
     256
     257   # File size and modification information...
     258   print "\nFile size: ", FormatFileSize($TextFilesInfo{FileSize}[$Index]), " \n";
     259   print "Last modified: ", $TextFilesInfo{FileLastModified}[$Index], " \n";
     260 }
     261
     262 # Total size of all the fiels...
     263 sub ListTotalSizeOfFiles {
     264   my($FileOkayCount, $TotalSize, $Index);
     265
     266   $FileOkayCount = 0;
     267   $TotalSize = 0;
     268
     269   for $Index (0 .. $#TextFilesList) {
     270     if ($TextFilesInfo{FileOkay}[$Index]) {
     271       $FileOkayCount++;
     272       $TotalSize += $TextFilesInfo{FileSize}[$Index];
     273     }
     274   }
     275   if ($FileOkayCount > 1) {
     276     print "\nTotal size of $FileOkayCount files: ", FormatFileSize($TotalSize), "\n";
     277   }
     278 }
     279
     280 # List data information...
     281 sub PrintDataInformation {
     282   my($InfoLabel, $DataLabelRef, $DataLabelToValueMapRef) = @_;
     283   my($Line, $Label);
     284
     285   $Line = "";
     286   for $Label (@{$DataLabelRef}) {
     287     $Line .= " \"$Label\" - " . (exists($DataLabelToValueMapRef->{$Label}) ? $DataLabelToValueMapRef->{$Label} : 0) . ",";
     288   }
     289   $Line =~ s/\,$//g;
     290   print "$InfoLabel: $Line\n";
     291 }
     292
     293 # Retrieve information about input text files...
     294 sub RetrieveTextFilesInfo {
     295   my($Index, $TextFile, $FileDir, $FileName, $FileExt, $InDelim, $Line, @ColLabels,  $ColNum, $ColLabel, $ModifiedTimeString, $ModifiedDateString);
     296
     297   %TextFilesInfo = ();
     298   @{$TextFilesInfo{FileOkay}} = ();
     299   @{$TextFilesInfo{ColCount}} = ();
     300   @{$TextFilesInfo{ColLabels}} = ();
     301   @{$TextFilesInfo{ColLabelToNumMap}} = ();
     302   @{$TextFilesInfo{InDelim}} = ();
     303   @{$TextFilesInfo{FileSize}} = ();
     304   @{$TextFilesInfo{FileLastModified}} = ();
     305
     306   FILELIST: for $Index (0 .. $#TextFilesList) {
     307     $TextFile = $TextFilesList[$Index];
     308
     309     $TextFilesInfo{FileOkay}[$Index] = 0;
     310     $TextFilesInfo{ColCount}[$Index] = 0;
     311     $TextFilesInfo{InDelim}[$Index] = "";
     312     $TextFilesInfo{FileSize}[$Index] = 0;
     313     $TextFilesInfo{FileLastModified}[$Index] = '';
     314     @{$TextFilesInfo{ColLabels}[$Index]} = ();
     315     %{$TextFilesInfo{ColLabelToNumMap}[$Index]} = ();
     316
     317     if (!(-e $TextFile)) {
     318       warn "Warning: Ignoring file $TextFile: It doesn't exist\n";
     319       next FILELIST;
     320     }
     321     if (!CheckFileType($TextFile, "csv tsv")) {
     322       warn "Warning: Ignoring file $TextFile: It's not a csv or tsv file\n";
     323       next FILELIST;
     324     }
     325     ($FileDir, $FileName, $FileExt) = ParseFileName($TextFile);
     326     if ($FileExt =~ /^tsv$/i) {
     327       $InDelim = "\t";
     328     }
     329     else {
     330       $InDelim = "\,";
     331       if ($OptionsInfo{InDelim} !~ /^(comma|semicolon)$/i) {
     332         warn "Warning: Ignoring file $TextFile: The value specified, $OptionsInfo{InDelim}, for option \"--indelim\" is not valid for csv files\n";
     333         next FILELIST;
     334       }
     335       if ($OptionsInfo{InDelim} =~ /^semicolon$/i) {
     336         $InDelim = "\;";
     337       }
     338     }
     339
     340     if (!open TEXTFILE, "$TextFile") {
     341       warn "Warning: Ignoring file $TextFile: Couldn't open it: $! \n";
     342       next FILELIST;
     343     }
     344
     345     $Line = GetTextLine(\*TEXTFILE);
     346     @ColLabels = quotewords($InDelim, 0, $Line);
     347     close TEXTFILE;
     348
     349     $TextFilesInfo{FileOkay}[$Index] = 1;
     350     $TextFilesInfo{InDelim}[$Index] = $InDelim;
     351
     352     $TextFilesInfo{ColCount}[$Index] = @ColLabels;
     353     push @{$TextFilesInfo{ColLabels}[$Index]}, @ColLabels;
     354     for $ColNum (0 .. $#ColLabels) {
     355       $ColLabel = $ColLabels[$ColNum];
     356       $TextFilesInfo{ColLabelToNumMap}[$Index]{$ColLabel} = $ColNum;
     357     }
     358     $TextFilesInfo{FileSize}[$Index] = FileSize($TextFile);
     359     ($ModifiedTimeString, $ModifiedDateString) = FormattedFileModificationTimeAndDate($TextFile);
     360     $TextFilesInfo{FileLastModified}[$Index] = "$ModifiedTimeString; $ModifiedDateString";
     361   }
     362
     363 }
     364
     365 # Make sure specified numerical data columns are okay...
     366 sub ProcessColumnsInfo {
     367   my($Index, $TextFile);
     368
     369   @{$TextFilesInfo{NumericalDataColNums}} = ();
     370   @{$TextFilesInfo{NumericalDataColLabels}} = ();
     371
     372   FILELIST: for $Index (0 .. $#TextFilesList) {
     373     $TextFile = $TextFilesList[$Index];
     374     @{$TextFilesInfo{NumericalDataColNums}[$Index]} = ();
     375     @{$TextFilesInfo{NumericalDataColLabels}[$Index]} = ();
     376
     377     if ($TextFilesInfo{FileOkay}[$Index]) {
     378       my($SpecifiedColNum, $ColNum, $ColLabel, @SpecifiedColNums, @SpecifiedColLabels);
     379       @SpecifiedColNums = ();
     380       if ($OptionsInfo{Mode} =~ /^colnum$/i) {
     381         for $SpecifiedColNum (@{$OptionsInfo{SpecifiedNumericalDataCols}}) {
     382           if ($SpecifiedColNum <= $TextFilesInfo{ColCount}[$Index]) {
     383             $ColNum = $SpecifiedColNum - 1;
     384             push @SpecifiedColNums, $ColNum;
     385             push @SpecifiedColLabels, $TextFilesInfo{ColLabels}[$Index][$ColNum];
     386           }
     387         }
     388       }
     389       else {
     390         for $ColLabel (@{$OptionsInfo{SpecifiedNumericalDataCols}}) {
     391           if (exists($TextFilesInfo{ColLabelToNumMap}[$Index]{$ColLabel})) {
     392             $ColNum = $TextFilesInfo{ColLabelToNumMap}[$Index]{$ColLabel};
     393             push @SpecifiedColNums, $ColNum;
     394             push @SpecifiedColLabels, $ColLabel;
     395           }
     396         }
     397       }
     398       if (@SpecifiedColNums) {
     399         push @{$TextFilesInfo{NumericalDataColNums}[$Index]}, @SpecifiedColNums;
     400         push @{$TextFilesInfo{NumericalDataColLabels}[$Index]}, @SpecifiedColLabels;
     401       }
     402     }
     403   }
     404 }
     405
     406 # Process option values...
     407 sub ProcessOptions {
     408   %OptionsInfo = ();
     409
     410   $OptionsInfo{Mode} = $Options{mode};
     411
     412   $OptionsInfo{All} = $Options{all} ? $Options{all} : 0;
     413   $OptionsInfo{Count} = $Options{count} ? $Options{count} : 0;
     414
     415   $OptionsInfo{DetailLevel} = $Options{detail} ? $Options{detail} : 1;
     416
     417   $OptionsInfo{Empty} = $Options{empty} ? $Options{empty} : 0;
     418
     419   $OptionsInfo{InDelim} = $Options{indelim};
     420   $OptionsInfo{NumericalDataCols} = $Options{numericaldatacols} ? $Options{numericaldatacols} : 0;
     421
     422   $OptionsInfo{ParseLines} = ($Options{all} || $Options{empty} || $Options{numericaldatacols}) ? 1 : 0;
     423   $OptionsInfo{CountEmpty} = ($Options{all} || $Options{empty}) ? 1 : 0;
     424   $OptionsInfo{CheckData} = ($Options{all} || $Options{datacheck}) ? 1 : 0;
     425   $OptionsInfo{CheckNumericalData} = ($Options{all} || $Options{numericaldatacols}) ? 1 : 0;
     426
     427   @{$OptionsInfo{SpecifiedNumericalDataCols}} = ();
     428   if ($Options{numericaldatacols}) {
     429     @{$OptionsInfo{SpecifiedNumericalDataCols}} = split ",", $Options{numericaldatacols};
     430     if ($Options{mode} =~ /^colnum$/i) {
     431       my($ColNum);
     432       for $ColNum (@{$OptionsInfo{SpecifiedNumericalDataCols}}) {
     433         if (!IsPositiveInteger($ColNum)) {
     434           die "Error: Invalid value $ColNum specified using \"--numericaldatacols\" option: Allowed values: > 0\n";
     435         }
     436       }
     437     }
     438   }
     439
     440 }
     441
     442 # Setup script usage  and retrieve command line arguments specified using various options...
     443 sub SetupScriptUsage {
     444
     445   # Retrieve all the options...
     446   %Options = ();
     447   $Options{detail} = 1;
     448   $Options{mode} = "colnum";
     449   $Options{indelim} = "comma";
     450   if (!GetOptions(\%Options, "all|a", "count|c", "datacheck", "detail|d=i", "empty|e", "help|h", "indelim=s", "mode|m=s", "numericaldatacols|n=s", "workingdir|w=s")) {
     451     die "\nTo get a list of valid options and their values, use \"$ScriptName -h\" or\n\"perl -S $ScriptName -h\" command and try again...\n";
     452   }
     453   if ($Options{workingdir}) {
     454     if (! -d $Options{workingdir}) {
     455       die "Error: The value specified, $Options{workingdir}, for option \"-w --workingdir\" is not a directory name.\n";
     456     }
     457     chdir $Options{workingdir} or die "Error: Couldn't chdir $Options{workingdir}: $! \n";
     458   }
     459   if ($Options{mode} !~ /^(colnum|collabel)$/i) {
     460     die "Error: The value specified, $Options{mode}, for option \"-m --mode\" is not valid. Allowed values: colnum or collabel\n";
     461   }
     462   if ($Options{indelim} !~ /^(comma|semicolon)$/i) {
     463     die "Error: The value specified, $Options{indelim}, for option \"--indelim\" is not valid. Allowed values: comma or semicolon\n";
     464   }
     465   if (!IsPositiveInteger($Options{detail})) {
     466     die "Error: The value specified, $Options{detail}, for option \"-d --detail\" is not valid. Allowed values: > 0\n";
     467   }
     468 }
     469