       1 #!/usr/bin/perl -w
       2 #
       3 # File: AnalyzeSDFilesData.pl
       4 # Author: Manish Sud <msud@san.rr.com>
       5 #
       6 # Copyright (C) 2018 Manish Sud. All rights reserved.
       7 #
       8 # This file is part of MayaChemTools.
       9 #
      10 # MayaChemTools is free software; you can redistribute it and/or modify it under
      11 # the terms of the GNU Lesser General Public License as published by the Free
      12 # Software Foundation; either version 3 of the License, or (at your option) any
      13 # later version.
      14 #
      15 # MayaChemTools is distributed in the hope that it will be useful, but without
      16 # any warranty; without even the implied warranty of merchantability of fitness
      17 # for a particular purpose.  See the GNU Lesser General Public License for more
      18 # details.
      19 #
      20 # You should have received a copy of the GNU Lesser General Public License
      21 # along with MayaChemTools; if not, see <http://www.gnu.org/licenses/> or
      22 # write to the Free Software Foundation Inc., 59 Temple Place, Suite 330,
      23 # Boston, MA, 02111-1307, USA.
      24 #
      25
      26 use strict;
      27 use FindBin; use lib "$FindBin::Bin/../lib";
      28 use Getopt::Long;
      29 use File::Basename;
      30 use Text::ParseWords;
      31 use Benchmark;
      32 use FileUtil;
      33 use SDFileUtil;
      34 use TextUtil;
      35 use StatisticsUtil;
      36
      37 my($ScriptName, %Options, $StartTime, $EndTime, $TotalTime);
      38
      39 # Autoflush STDOUT
      40 $| = 1;
      41
      42 # Starting message...
      43 $ScriptName = basename($0);
      44 print "\n$ScriptName: Starting...\n\n";
      45 $StartTime = new Benchmark;
      46
      47 # Get the options and setup script...
      48 SetupScriptUsage();
      49 if ($Options{help} || @ARGV < 1) {
      50   die GetUsageFromPod("$FindBin::Bin/$ScriptName");
      51 }
      52
      53 my(@SDFilesList);
      54 @SDFilesList = ExpandFileNames(\@ARGV, "sd sdf");
      55
      56 print "Processing options...\n";
      57 my(%OptionsInfo);
      58 ProcessOptions();
      59
      60 # Collect information about SD files...
      61 print "Checking input SD file(s)...\n";
      62 my(%SDFilesInfo);
      63 RetrieveSDFilesInfo();
      64 ProcessSDFilesDataLabelsInfo();
      65
      66 # Generate output files...
      67 my($FileIndex);
      68 if (@SDFilesList > 1) {
      69   print "\nProcessing SD files...\n";
      70 }
      71 for $FileIndex (0 .. $#SDFilesList) {
      72   if ($SDFilesInfo{FileOkay}[$FileIndex]) {
      73     print "\nProcessing file $SDFilesList[$FileIndex]...\n";
      74     AnalyzeSDFile($FileIndex);
      75   }
      76 }
      77 print "\n$ScriptName:Done...\n\n";
      78
      79 $EndTime = new Benchmark;
      80 $TotalTime = timediff ($EndTime, $StartTime);
      81 print "Total time: ", timestr($TotalTime), "\n";
      82
      83 ###############################################################################
      84
      85 # Analyze data...
      86 sub AnalyzeSDFile {
      87   my($Index) = @_;
      88   my($SDFile, $DataLabel, $DataValue, @DataLabelsToAnalyze, %DataFieldValuesToAnalyzeMap);
      89
      90   $SDFile = $SDFilesList[$Index];
      91   @DataLabelsToAnalyze = @{$SDFilesInfo{UniqueDataLabelsToAnalyze}[$Index]};
      92   %DataFieldValuesToAnalyzeMap = ();
      93   for $DataLabel (@DataLabelsToAnalyze) {
      94     @{$DataFieldValuesToAnalyzeMap{$DataLabel}} = ();
      95   }
      96
      97   # Collect appropriate data field label values for analysis...
      98   my($CmpdString, @CmpdLines, %DataFieldValues, $CmpdCount, $InvalidCmpdCount, @InvalidCmpdDataLabels);
      99   open SDFILE, "$SDFile" or die "Error: Can't open $SDFile: $! \n";
     100   $CmpdCount = 0;
     101   $InvalidCmpdCount = 0;
     102   while ($CmpdString = ReadCmpdString(\*SDFILE)) {
     103     $CmpdCount++;
     104     @CmpdLines = split "\n", $CmpdString;
     105     %DataFieldValues = GetCmpdDataHeaderLabelsAndValues(\@CmpdLines);
     106     @InvalidCmpdDataLabels = ();
     107     DATALABEL: for $DataLabel (@DataLabelsToAnalyze) {
     108       if (exists $DataFieldValues{$DataLabel}) {
     109         $DataValue = $DataFieldValues{$DataLabel};
     110         if ($OptionsInfo{CheckData}) {
     111           if (!IsNumerical($DataValue)) {
     112             push @InvalidCmpdDataLabels, $DataLabel;
     113             next DATALABEL;
     114           }
     115         }
     116         push @{$DataFieldValuesToAnalyzeMap{$DataLabel}}, $DataValue;
     117       }
     118     }
     119     if (@InvalidCmpdDataLabels) {
     120       $InvalidCmpdCount++;
     121       if ($OptionsInfo{DetailLevel} >=4 ) {
     122         print "Compound record $CmpdCount contains ", scalar(@InvalidCmpdDataLabels)," non-numerical or empty value(s) for data field(s) - ", JoinWords(\@InvalidCmpdDataLabels, ", ", 0)," - to be analyzed:\n$CmpdString \n";
     123       }
     124       elsif ($OptionsInfo{DetailLevel} >= 3) {
     125         print "Compound record $CmpdCount contains ", scalar(@InvalidCmpdDataLabels)," non-numerical or empty value(s) for data field(s) - ", JoinWords(\@InvalidCmpdDataLabels, ", ", 0)," - to be analyzed...\n";
     126       }
     127       elsif ($OptionsInfo{DetailLevel} >= 2) {
     128         print "Compound record $CmpdCount contains ", scalar(@InvalidCmpdDataLabels)," non-numerical or empty value(s) for data field to be analyzed...\n";
     129       }
     130     }
     131   }
     132   if ($InvalidCmpdCount && ($OptionsInfo{DetailLevel} >= 1)) {
     133     print "Non-numerical or empty data present in $InvalidCmpdCount compound record(s)...\n";
     134   }
     135   close SDFILE;
     136
     137   # Perform the analysis...
     138   my(@SpecifiedFunctionNames, $SpecifiedFunction);
     139   @SpecifiedFunctionNames = ();
     140
     141   for $SpecifiedFunction (@{$OptionsInfo{SpecifiedStatisticalFunctions}}) {
     142     if ($SpecifiedFunction !~ /^(Covariance|Correlation|Frequency|Rsquare|StandardScores|StandardScoresN)$/i) {
     143       push @SpecifiedFunctionNames, $OptionsInfo{SpecifiedStatisticalFunctionsMap}{lc($SpecifiedFunction)};
     144     }
     145   }
     146   if (@SpecifiedFunctionNames) {
     147     PerformAnalysis($Index, \@SpecifiedFunctionNames, \%DataFieldValuesToAnalyzeMap)
     148   }
     149   if (exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{covariance}) || exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{correlation}) || exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{rsquare})) {
     150     if ($OptionsInfo{AllDataLabelPairs} || $OptionsInfo{CommonDataLabelPairs}) {
     151       PerformMatrixAnalysis($Index, \%DataFieldValuesToAnalyzeMap);
     152     }
     153     else {
     154       # Perform pairwise analysis for specified columns and write out calculated values - correlation
     155       # rsquare, or covariance - in the same file.
     156       PerformDataLabelPairAnalysis($Index, \%DataFieldValuesToAnalyzeMap);
     157     }
     158   }
     159   if (exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{standardscores}) || exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{standardscoresn}) ) {
     160     PerformStandardScoresAnalysis($Index, \%DataFieldValuesToAnalyzeMap);
     161   }
     162   if (exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{frequency})) {
     163     PerformFrequencyAnalysis($Index, \%DataFieldValuesToAnalyzeMap);
     164   }
     165
     166 }
     167
     168 # Calculate values for various statistical functions...
     169 sub PerformAnalysis {
     170   my($Index, $SpecifiedFunctionNamesRef, $DataValuesToAnalyzeMapRef) = @_;
     171   my($NewTextFile, $Line, $SpecifiedFunction, $Label, @ColLabels, @DataLabelsToAnalyze);
     172
     173   $NewTextFile = $SDFilesInfo{NewTextFileRoot}[$Index] . $OptionsInfo{FileNameMode} . "." . $SDFilesInfo{NewTextFileExt}[$Index];
     174
     175   print "Generating new text file $NewTextFile...\n";
     176   open NEWTEXTFILE, ">$NewTextFile" or die "Error: Can't open $NewTextFile: $! \n";
     177
     178   # Write out column labels...
     179   @ColLabels = ();
     180   push @ColLabels, "DataLabel";
     181   for $SpecifiedFunction (@{$SpecifiedFunctionNamesRef}) {
     182     $Label = $SpecifiedFunction;
     183     if ($SpecifiedFunction =~ /^(KLargest|KSmallest)$/i) {
     184       my($KthValue);
     185       $KthValue = ($SpecifiedFunction =~ /^KLargest$/i) ? $OptionsInfo{KLargest} : $OptionsInfo{KSmallest};
     186       $Label = AddNumberSuffix($KthValue) . "$SpecifiedFunction";
     187       $Label =~ s/K//g;
     188     }
     189     elsif ($SpecifiedFunction =~ /^TrimMean$/i) {
     190       $Label = "${SpecifiedFunction}($OptionsInfo{TrimFraction})";
     191     }
     192     push @ColLabels, $Label;
     193   }
     194   $Line = JoinWords(\@ColLabels, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
     195   print NEWTEXTFILE "$Line\n";
     196
     197   # Go over each column to be analyzed...
     198   @DataLabelsToAnalyze = @{$SDFilesInfo{DataLabelsToAnalyze}[$Index]};
     199
     200   # Turn off "strict"; otherwise, invoking statistical functions using function name string
     201   # is problematic.
     202   no strict;
     203
     204   my($DataValuesRef, $DataLabel, $Value, @RowValues, %CalculatedValues);
     205   %CalculatedValues = ();
     206   for $DataLabel (@DataLabelsToAnalyze) {
     207     @RowValues = ();
     208     # Setup column id...
     209     push @RowValues, $DataLabel;
     210     $DataValuesRef =  \@{$DataValuesToAnalyzeMapRef->{$DataLabel}};
     211     FUNCTIONNAME: for $SpecifiedFunction (@{$SpecifiedFunctionNamesRef}) {
     212       $Value = "";
     213       if (!@{$DataValuesToAnalyzeMapRef->{$DataLabel}}) {
     214         # Invalid column values...
     215         push @RowValues, $Value;
     216         next FUNCTIONNAME;
     217       }
     218       if ($SpecifiedFunction =~ /^Count$/i) {
     219         $Value = @{$DataValuesToAnalyzeMapRef->{$DataLabel}};
     220       }
     221       elsif ($SpecifiedFunction =~ /^KLargest$/i) {
     222         $Value = &$SpecifiedFunction($DataValuesRef, $OptionsInfo{KLargest});
     223       }
     224       elsif ($SpecifiedFunction =~ /^KSmallest$/i) {
     225         $Value = &$SpecifiedFunction($DataValuesRef, $OptionsInfo{KSmallest});
     226       }
     227       elsif ($SpecifiedFunction =~ /^StandardDeviation$/i) {
     228         if (exists($CalculatedValues{$DataLabel}{StandardDeviation})) {
     229           $Value = $CalculatedValues{$DataLabel}{StandardDeviation};
     230         }
     231         else {
     232           $Value = &$SpecifiedFunction($DataValuesRef);
     233           $CalculatedValues{$DataLabel}{StandardDeviation} = $Value;
     234         }
     235       }
     236       elsif ($SpecifiedFunction =~ /^StandardError$/i) {
     237         if (!exists($CalculatedValues{$DataLabel}{StandardDeviation})) {
     238           $Value = StandardDeviation($DataValuesRef);
     239           $CalculatedValues{$DataLabel}{StandardDeviation} = $Value;
     240         }
     241         if (defined $CalculatedValues{$DataLabel}{StandardDeviation}) {
     242           $Value = &$SpecifiedFunction($CalculatedValues{$DataLabel}{StandardDeviation}, @{$DataValuesToAnalyzeMapRef->{$DataLabel}});
     243         }
     244       }
     245       elsif ($SpecifiedFunction =~ /^TrimMean$/i) {
     246         $Value = &$SpecifiedFunction($DataValuesRef, $OptionsInfo{TrimFraction});
     247       }
     248       else {
     249         $Value = &$SpecifiedFunction($DataValuesRef);
     250       }
     251       # Format the output value. And add zero to get rid of tariling zeros...
     252       $Value = (defined($Value) && length($Value)) ? (sprintf("%.$OptionsInfo{Precision}f", $Value) + 0) : "";
     253       push @RowValues, $Value;
     254     }
     255     $Line = JoinWords(\@RowValues, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
     256     print NEWTEXTFILE "$Line\n";
     257   }
     258   close NEWTEXTFILE;
     259 }
     260
     261 # Calculate covariance, correlation, rsquare for specified data field label pairs....
     262 sub PerformDataLabelPairAnalysis {
     263   my($Index, $DataValuesToAnalyzeMapRef) = @_;
     264   my($NewTextFile, @ColLabels, $Line, $CalculateCorrelation, $CalculateRSquare, $CalculateCovariance);
     265
     266   $CalculateCorrelation = exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{correlation}) ? 1 : 0;
     267   $CalculateRSquare = exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{rsquare}) ? 1 : 0;
     268   $CalculateCovariance = exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{covariance}) ? 1 : 0;
     269
     270   $NewTextFile = $SDFilesInfo{NewTextFileRoot}[$Index] . "DataFieldPairsAnalysis." .  $SDFilesInfo{NewTextFileExt}[$Index];
     271   print "Generating new text file $NewTextFile...\n";
     272   open NEWTEXTFILE, ">$NewTextFile" or die "Error: Can't open $NewTextFile: $! \n";
     273
     274   # Write out the column labels...
     275   @ColLabels = ();
     276   push @ColLabels, ("DataLabel1", "DataLabel2");
     277   if ($CalculateCorrelation || $CalculateRSquare) {
     278     push @ColLabels, "Correlation";
     279     if ($CalculateRSquare) {
     280       push @ColLabels, "RSquare";
     281     }
     282   }
     283   if ($CalculateCovariance) {
     284     push @ColLabels, "Covariance";
     285   }
     286   $Line = JoinWords(\@ColLabels, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
     287   print NEWTEXTFILE "$Line\n";
     288
     289   # Go over each data field pair...
     290   my($CorrelationValue, $RSquareValue, $CovarianceValue,  $LabelIndex, $DataLabel1, $DataLabel2, $DataValues1, $DataValues2, @DataLabelPairs1ToAnalyze, @DataLabelPairs2ToAnalyze, @RowValues, $Value);
     291
     292   @DataLabelPairs1ToAnalyze = @{$SDFilesInfo{DataLabelPairs1ToAnalyze}[$Index]};
     293   @DataLabelPairs2ToAnalyze = @{$SDFilesInfo{DataLabelPairs2ToAnalyze}[$Index]};
     294   for $LabelIndex (0 .. $#DataLabelPairs1ToAnalyze) {
     295     @RowValues = ();
     296     $DataLabel1 = $DataLabelPairs1ToAnalyze[$LabelIndex];
     297     $DataLabel2 = $DataLabelPairs2ToAnalyze[$LabelIndex];
     298     $DataValues1 =  \@{$DataValuesToAnalyzeMapRef->{$DataLabel1}};
     299     $DataValues2 =  \@{$DataValuesToAnalyzeMapRef->{$DataLabel2}};
     300
     301     # Setup column ids...
     302     push @RowValues, $DataLabel1;
     303     push @RowValues, $DataLabel2;
     304
     305     if (@$DataValues1 != @$DataValues2) {
     306       # Print a warning...
     307       warn "Warning: Skipping analysis for data field pair $DataLabel1, $DataLabel2: Number of valid data values must be same.\n";
     308       if ($CalculateCorrelation || $CalculateRSquare) {
     309         push @RowValues, "";
     310         if ($CalculateRSquare) {
     311           push @RowValues, "";
     312         }
     313       }
     314       if ($CalculateCovariance) {
     315         push @RowValues, "";
     316       }
     317     }
     318     else {
     319       # Calculate appropriate value...
     320       if ($CalculateCorrelation || $CalculateRSquare) {
     321         $CorrelationValue = Correlation($DataValues1, $DataValues2);
     322         $Value = (defined($CorrelationValue) && length($CorrelationValue)) ? (sprintf("%.$OptionsInfo{Precision}f", $CorrelationValue) + 0) : "";
     323         push @RowValues, $Value;
     324         if ($CalculateRSquare) {
     325           $RSquareValue = (defined($CorrelationValue) && length($CorrelationValue)) ? ($CorrelationValue ** 2) : "";
     326           $Value = (length($RSquareValue)) ? (sprintf("%.$OptionsInfo{Precision}f", $RSquareValue) + 0) : "";
     327           push @RowValues, $Value;
     328         }
     329       }
     330       if ($CalculateCovariance) {
     331         $CovarianceValue = Covariance($DataValues1, $DataValues2);
     332         $Value = (defined($CovarianceValue) && length($CovarianceValue)) ? (sprintf("%.$OptionsInfo{Precision}f", $CovarianceValue) + 0) : "";
     333         push @RowValues, $Value;
     334       }
     335     }
     336     $Line = JoinWords(\@RowValues, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
     337     print NEWTEXTFILE "$Line\n";
     338   }
     339   close NEWTEXTFILE;
     340 }
     341
     342 # Generate histogram numbers...
     343 sub PerformFrequencyAnalysis {
     344   my($Index, $DataValuesToAnalyzeMapRef) = @_;
     345   my($NewTextFile, $ColLabel, @ColLabels, @RowValues, $Line, $DataLabel, @DataLabelsToAnalyze, $DataValuesRef, $BinValue, $FrequencyValue, $Value, %FrequencyMap);
     346
     347   @DataLabelsToAnalyze = @{$SDFilesInfo{DataLabelsToAnalyze}[$Index]};
     348   for $DataLabel (@DataLabelsToAnalyze) {
     349     $NewTextFile = $SDFilesInfo{NewTextFileRoot}[$Index] . $DataLabel . "FrequencyAnalysis." .  $SDFilesInfo{NewTextFileExt}[$Index];
     350     print "Generating new text file $NewTextFile...\n";
     351     open NEWTEXTFILE, ">$NewTextFile" or die "Error: Can't open $NewTextFile: $! \n";
     352
     353     # Write out the column labels...
     354     @ColLabels = ();
     355     push @ColLabels , ("Bins", "Frequency");
     356     $Line = JoinWords(\@ColLabels, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
     357     print NEWTEXTFILE "$Line\n";
     358
     359     #Calculate and write out frequency values...
     360     %FrequencyMap = ();
     361     $DataValuesRef =  \@{$DataValuesToAnalyzeMapRef->{$DataLabel}};
     362     if (@$DataValuesRef) {
     363       if (@{$OptionsInfo{BinRange}}) {
     364         %FrequencyMap = Frequency($DataValuesRef, \@{$OptionsInfo{BinRange}});
     365       }
     366       else {
     367         %FrequencyMap = Frequency($DataValuesRef, $OptionsInfo{NumOfBins});
     368       }
     369     }
     370     for $BinValue (sort { $a <=> $b }  keys %FrequencyMap) {
     371       $FrequencyValue = $FrequencyMap{$BinValue};
     372
     373       @RowValues = ();
     374       $Value = (length($BinValue)) ? (sprintf("%.$OptionsInfo{Precision}f", $BinValue) + 0) : "";
     375       push @RowValues, $Value;
     376       $Value = (length($FrequencyValue)) ? (sprintf("%.$OptionsInfo{Precision}f", $FrequencyValue) + 0) : "";
     377       push @RowValues, $Value;
     378
     379       $Line = JoinWords(\@RowValues, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
     380       print NEWTEXTFILE "$Line\n";
     381     }
     382     close NEWTEXTFILE;
     383   }
     384 }
     385
     386 # Calculate covariance, correlation/rsquare matrices....
     387 sub PerformMatrixAnalysis {
     388   my($Index, $DataValuesToAnalyzeMapRef) = @_;
     389   my($CorrelationTextFile, $CovarianceTextFile, $RSquareTextFile, $CalculateCorrelation, $CalculateRSquare, $CalculateCovariance);
     390
     391   $CalculateCorrelation = exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{correlation}) ? 1 : 0;
     392   $CalculateRSquare = exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{rsquare}) ? 1 : 0;
     393   $CalculateCovariance = exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{covariance}) ? 1 : 0;
     394
     395   $CorrelationTextFile = $SDFilesInfo{NewTextFileRoot}[$Index] . "CorrelationMatrix." .  $SDFilesInfo{NewTextFileExt}[$Index];
     396   $RSquareTextFile = $SDFilesInfo{NewTextFileRoot}[$Index] . "RSquareMatrix." .  $SDFilesInfo{NewTextFileExt}[$Index];
     397   $CovarianceTextFile = $SDFilesInfo{NewTextFileRoot}[$Index] . "CovarianceMatrix." .  $SDFilesInfo{NewTextFileExt}[$Index];
     398
     399   my($TextFilesList, $Delimiter);
     400   $TextFilesList =  "";
     401   if ($CalculateCorrelation || $CalculateRSquare) {
     402     $TextFilesList = $CorrelationTextFile;
     403     if ($CalculateRSquare) {
     404       $TextFilesList .= ", $CorrelationTextFile";
     405     }
     406   }
     407   $Delimiter = length($TextFilesList) ? "," : "";
     408   if ($CalculateCovariance) {
     409     $TextFilesList .= "${Delimiter} ${CorrelationTextFile}";
     410   }
     411   if ($TextFilesList =~ /\,/) {
     412     print "Generating new text files $TextFilesList...\n"
     413   }
     414   else {
     415     print "Generating new text file $TextFilesList...\n"
     416   }
     417   if ($CalculateCorrelation || $CalculateRSquare) {
     418     open CORRELATIONTEXTFILE, ">$CorrelationTextFile" or die "Error: Can't open $CorrelationTextFile: $! \n";
     419     if ($CalculateRSquare) {
     420       open RSQUARETEXTFILE, ">$RSquareTextFile" or die "Error: Can't open $RSquareTextFile: $! \n";
     421     }
     422   }
     423   if ($CalculateCovariance) {
     424     open COVARIANCETEXTFILE, ">$CovarianceTextFile" or die "Error: Can't open $CovarianceTextFile: $! \n";
     425   }
     426
     427   my($Line, $Value, $CorrelationValue, $RSquareValue, $CovarianceValue, $DataLabel, $DataLabel1, $DataLabel2, $DataValuesRef1, $DataValuesRef2, @ColLabels, @CovarianceRowValues, @CorrelationRowValues, @RSquareRowValues);
     428
     429   # Write out the column labels...
     430   @ColLabels = ();
     431   push @ColLabels, @{$SDFilesInfo{AllDataLabels}[$Index]};
     432   $Line = JoinWords(\@ColLabels, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
     433   if ($CalculateCorrelation || $CalculateRSquare) {
     434     print CORRELATIONTEXTFILE "$Line\n";
     435     if ($CalculateRSquare) {
     436       print RSQUARETEXTFILE "$Line\n";
     437     }
     438   }
     439   if ($CalculateCovariance) {
     440     print COVARIANCETEXTFILE "$Line\n";
     441   }
     442
     443   # Due to symmetric nature of these matrices, only one half needs to be
     444   # calculated. So, just calculate the lower half and copy it to upper half...
     445   my(%CorrelationMatrixMap, %RSquareMatrixMap, %CovarianceMatrixMap, $LabelIndex1, $LabelIndex2, @DataLabelsToAnalyze);
     446
     447   %CorrelationMatrixMap = (); %RSquareMatrixMap = (); %CovarianceMatrixMap = ();
     448   @DataLabelsToAnalyze = ();
     449   @DataLabelsToAnalyze = $OptionsInfo{AllDataLabelPairs} ? @{$SDFilesInfo{AllDataLabels}[$Index]} : @{$SDFilesInfo{CommonDataLabels}[$Index]};
     450
     451   for $LabelIndex1 (0 .. (@DataLabelsToAnalyze - 1)) {
     452     $DataLabel1 = $DataLabelsToAnalyze[$LabelIndex1];
     453     for $LabelIndex2 (0 .. $LabelIndex1) {
     454       $DataLabel2 = $DataLabelsToAnalyze[$LabelIndex2];
     455       $DataValuesRef1 =  \@{$DataValuesToAnalyzeMapRef->{$DataLabel1}};
     456       $DataValuesRef2 =  \@{$DataValuesToAnalyzeMapRef->{$DataLabel2}};
     457       if ($CalculateCorrelation || $CalculateRSquare) {
     458         $CorrelationValue = Correlation($DataValuesRef1, $DataValuesRef2);
     459         $CorrelationValue = (defined($CorrelationValue) && length($CorrelationValue)) ? (sprintf("%.$OptionsInfo{Precision}f", $CorrelationValue) + 0) : "";
     460         $CorrelationMatrixMap{$DataLabel1}{$DataLabel2} = $CorrelationValue;
     461         if ($DataLabel1 ne $DataLabel2) {
     462           $CorrelationMatrixMap{$DataLabel2}{$DataLabel1} = $CorrelationValue;
     463         }
     464         if ($CalculateRSquare) {
     465           $RSquareValue = (defined($CorrelationValue) && length($CorrelationValue)) ? ($CorrelationValue ** 2) : "";
     466           $RSquareValue = (length($RSquareValue)) ? (sprintf("%.$OptionsInfo{Precision}f", $RSquareValue) + 0) : "";
     467           $RSquareMatrixMap{$DataLabel1}{$DataLabel2} = $RSquareValue;
     468           if ($DataLabel1 ne $DataLabel2) {
     469             $RSquareMatrixMap{$DataLabel2}{$DataLabel1} = $RSquareValue;
     470           }
     471         }
     472       }
     473       if ($CalculateCovariance) {
     474         $CovarianceValue = Covariance($DataValuesRef1, $DataValuesRef2);
     475         $CovarianceValue = (defined($CovarianceValue) && length($CovarianceValue)) ? (sprintf("%.$OptionsInfo{Precision}f", $CovarianceValue) + 0) : "";
     476         $CovarianceMatrixMap{$DataLabel1}{$DataLabel2} = $CovarianceValue;
     477         if ($DataLabel1 ne $DataLabel2) {
     478           $CovarianceMatrixMap{$DataLabel2}{$DataLabel1} = $CovarianceValue;
     479         }
     480       }
     481     }
     482   }
     483
     484   # Write out the matrices...
     485   for $LabelIndex1 (0 .. (@DataLabelsToAnalyze - 1)) {
     486     $DataLabel1 = $DataLabelsToAnalyze[$LabelIndex1];
     487     @CorrelationRowValues = ();
     488     @RSquareRowValues = ();
     489     @CovarianceRowValues = ();
     490     if ($CalculateCorrelation || $CalculateRSquare) {
     491       push @CorrelationRowValues, $DataLabel1;
     492       if ($CalculateRSquare) {
     493         push @RSquareRowValues, $DataLabel1;
     494       }
     495     }
     496     if ($CalculateCovariance) {
     497       push @CovarianceRowValues, $DataLabel;
     498     }
     499     for $LabelIndex2 (0 .. (@DataLabelsToAnalyze - 1)) {
     500       $DataLabel2 = $DataLabelsToAnalyze[$LabelIndex2];
     501       if ($CalculateCorrelation || $CalculateRSquare) {
     502         push @CorrelationRowValues, $CorrelationMatrixMap{$DataLabel1}{$DataLabel2};
     503         if ($CalculateRSquare) {
     504           push @RSquareRowValues, $RSquareMatrixMap{$DataLabel1}{$DataLabel2};
     505         }
     506       }
     507       if ($CalculateCovariance) {
     508         push @CovarianceRowValues, $CovarianceMatrixMap{$DataLabel1}{$DataLabel2};
     509       }
     510     }
     511     if ($CalculateCorrelation || $CalculateRSquare) {
     512       $Line = JoinWords(\@CorrelationRowValues, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
     513       print CORRELATIONTEXTFILE "$Line\n";
     514       if ($CalculateRSquare) {
     515         $Line = JoinWords(\@RSquareRowValues, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
     516         print RSQUARETEXTFILE "$Line\n";
     517       }
     518     }
     519     if ($CalculateCovariance) {
     520       $Line = JoinWords(\@CovarianceRowValues, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
     521       print COVARIANCETEXTFILE "$Line\n";
     522     }
     523   }
     524   if ($CalculateCorrelation || $CalculateRSquare) {
     525     close CORRELATIONTEXTFILE;
     526     if ($CalculateRSquare) {
     527       close RSQUARETEXTFILE;
     528     }
     529   }
     530   if ($CalculateCovariance) {
     531     close COVARIANCETEXTFILE;
     532   }
     533 }
     534
     535 # Calculate standard scores...
     536 sub PerformStandardScoresAnalysis {
     537   my($Index, $DataValuesToAnalyzeMapRef) = @_;
     538   my($StandardScores, $StandardScoresN, $NewTextFile, @ColLabels, $Label, $NewLine);
     539
     540   $StandardScores = exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{standardscores}) ? 1 : 0;
     541   $StandardScoresN = exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{standardscoresn}) ? 1 : 0;
     542
     543   $NewTextFile = $SDFilesInfo{NewTextFileRoot}[$Index] . "StandardScores." .  $SDFilesInfo{NewTextFileExt}[$Index];
     544   print "Generating new text file $NewTextFile...\n";
     545   open NEWTEXTFILE, ">$NewTextFile" or die "Error: Can't open $NewTextFile: $! \n";
     546
     547   my($DataLabel, @DataLabelsToAnalyze);
     548   # Write out column labels...
     549   @ColLabels = ();
     550   @DataLabelsToAnalyze = @{$SDFilesInfo{DataLabelsToAnalyze}[$Index]};
     551   for $DataLabel (@DataLabelsToAnalyze) {
     552     if ($StandardScores) {
     553       push @ColLabels, "${DataLabel}\(StandardScores)";
     554     }
     555     if ($StandardScoresN) {
     556       push @ColLabels, "${DataLabel}\(StandardScoresN)";
     557     }
     558   }
     559   $NewLine = JoinWords(\@ColLabels, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
     560   print NEWTEXTFILE "$NewLine\n";
     561
     562   # Go over each column to be analyzed and calculate standard deviation
     563   # and mean values...
     564   my($DataValuesRef, %StandardDeviationMap, %StandardDeviationNMap, %MeanMap);
     565   %StandardDeviationMap = ();
     566   %StandardDeviationNMap = ();
     567   %MeanMap = ();
     568   for $DataLabel (@DataLabelsToAnalyze) {
     569     $DataValuesRef =  \@{$DataValuesToAnalyzeMapRef->{$DataLabel}};
     570     if (!exists($MeanMap{$DataLabel})) {
     571       $MeanMap{$DataLabel} = Mean($DataValuesRef);
     572     }
     573     if ($StandardScores) {
     574       if (!exists($StandardDeviationMap{$DataLabel})) {
     575         $StandardDeviationMap{$DataLabel} = StandardDeviation($DataValuesRef);
     576       }
     577     }
     578     if ($StandardScoresN) {
     579       if (!exists($StandardDeviationNMap{$DataLabel})) {
     580         $StandardDeviationNMap{$DataLabel} = StandardDeviationN($DataValuesRef);
     581       }
     582     }
     583   }
     584   #
     585   # Go over each data field and calculate standard scores for each column
     586   # using (x[i] - mean) / (n - 1) for StandardScores and (x[i] - mean) / n
     587   # for StandardScoresN; write out the calculated values as well...
     588
     589   my($SDFile, $Value, $ValueOkay, $ScoreValue, @RowValues, $CmpdString, @CmpdLines, %DataFieldValues);
     590   $SDFile = $SDFilesList[$Index];
     591
     592   open SDFILE, "$SDFile" or die "Error: Can't open $SDFile: $! \n";
     593   while ($CmpdString = ReadCmpdString(\*SDFILE)) {
     594     @CmpdLines = split "\n", $CmpdString;
     595     %DataFieldValues = GetCmpdDataHeaderLabelsAndValues(\@CmpdLines);
     596     @RowValues = ();
     597     for $DataLabel (@DataLabelsToAnalyze) {
     598       $Value = "";
     599       if (exists $DataFieldValues{$DataLabel}) {
     600         $Value = $DataFieldValues{$DataLabel};
     601       }
     602       $ValueOkay = ($OptionsInfo{CheckData} && !IsNumerical($Value)) ? 0 : 1;
     603       if ($StandardScores) {
     604         $ScoreValue = $ValueOkay ? (($Value - $MeanMap{$DataLabel})/$StandardDeviationMap{$DataLabel}) : "";
     605         $ScoreValue = (defined($ScoreValue) && length($ScoreValue)) ? (sprintf("%.$OptionsInfo{Precision}f", $ScoreValue) + 0) : "";
     606         push @RowValues, $ScoreValue;
     607       }
     608       if ($StandardScoresN) {
     609         $ScoreValue = $ValueOkay ? (($Value - $MeanMap{$DataLabel})/$StandardDeviationNMap{$DataLabel}) : "";
     610         $ScoreValue = (defined($ScoreValue) && length($ScoreValue)) ? (sprintf("%.$OptionsInfo{Precision}f", $ScoreValue) + 0) : "";
     611         push @RowValues, $ScoreValue;
     612       }
     613     }
     614     $NewLine = JoinWords(\@RowValues, $OptionsInfo{OutDelim}, $OptionsInfo{OutQuote});
     615     print NEWTEXTFILE "$NewLine\n";
     616   }
     617   close SDFILE;
     618   close NEWTEXTFILE;
     619
     620 }
     621
     622 # Make sure the specified data field labels exists in SD files...
     623 sub ProcessSDFilesDataLabelsInfo {
     624   my($Index, $DataFieldIndex, $SDFile, $DataLabel, @DataLabelsToAnalyze, %UniqueDataLabelsToAnalyzeMap);
     625
     626   @{$SDFilesInfo{DataLabelsToAnalyze}} = ();
     627   @{$SDFilesInfo{DataLabelPairs1ToAnalyze}} = ();
     628   @{$SDFilesInfo{DataLabelPairs2ToAnalyze}} = ();
     629   @{$SDFilesInfo{UniqueDataLabelsToAnalyze}} = ();
     630
     631   FILELIST: for $Index (0 .. $#SDFilesList) {
     632     $SDFile = $SDFilesList[$Index];
     633
     634     @{$SDFilesInfo{DataLabelsToAnalyze}[$Index]} = ();
     635     @{$SDFilesInfo{DataLabelPairs1ToAnalyze}[$Index]} = ();
     636     @{$SDFilesInfo{DataLabelPairs2ToAnalyze}[$Index]} = ();
     637     @{$SDFilesInfo{UniqueDataLabelsToAnalyze}[$Index]} = ();
     638
     639     %UniqueDataLabelsToAnalyzeMap = ();
     640
     641     if ($SDFilesInfo{FileOkay}[$Index]) {
     642       @DataLabelsToAnalyze = ();
     643       if (@{$OptionsInfo{SpecifiedDataLabels}}) {
     644         for $DataLabel (@{$OptionsInfo{SpecifiedDataLabels}}) {
     645           if (exists($SDFilesInfo{AllDataLabelsMap}[$Index]{$DataLabel})) {
     646             push @DataLabelsToAnalyze, $DataLabel;
     647           }
     648         }
     649       }
     650       elsif (defined($OptionsInfo{DataFields}) && $OptionsInfo{DataFields} =~ /^All$/i) {
     651         push @DataLabelsToAnalyze, @{$SDFilesInfo{AllDataLabels}[$Index]};
     652       }
     653       else {
     654         push @DataLabelsToAnalyze, @{$SDFilesInfo{CommonDataLabels}[$Index]};
     655       }
     656       if (@DataLabelsToAnalyze) {
     657         push @{$SDFilesInfo{DataLabelsToAnalyze}[$Index]}, @DataLabelsToAnalyze;
     658         # Set up unique data field label map as well...
     659         for $DataLabel (@DataLabelsToAnalyze) {
     660           if (!exists $UniqueDataLabelsToAnalyzeMap{$DataLabel}) {
     661             $UniqueDataLabelsToAnalyzeMap{$DataLabel} = $DataLabel;
     662           }
     663         }
     664       }
     665       else {
     666         warn "Warning: Ignoring file $SDFile: None of the data field labels specified, @{$OptionsInfo{SpecifiedDataLabels}}, using \"--datafields\" option exist.\n";
     667         $SDFilesInfo{FileOkay}[$Index] = 0;
     668         next FILELIST;
     669       }
     670       if (!$OptionsInfo{Overwrite} && exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{frequency})) {
     671         # Make sure specific frequency files don't exist...
     672         my($FrequencyFile);
     673         for $DataLabel (@DataLabelsToAnalyze) {
     674           $FrequencyFile = $SDFilesInfo{NewTextFileRoot}[$Index] . $SDFilesInfo{AllDataLabelsMap}[$Index]{$DataLabel} . "FrequencyAnalysis." .  $SDFilesInfo{NewTextFileExt}[$Index];
     675           if (-e $FrequencyFile) {
     676             warn "Warning: Ignoring file $SDFile: The file $FrequencyFile already exists.\n";
     677             $SDFilesInfo{FileOkay}[$Index] = 0;
     678             next FILELIST;
     679           }
     680         }
     681       }
     682       # Setup specified data field label pairs...
     683       if (exists $OptionsInfo{SpecifiedStatisticalFunctionsMap}{correlation} || exists $OptionsInfo{SpecifiedStatisticalFunctionsMap}{covariance} || exists $OptionsInfo{SpecifiedStatisticalFunctionsMap}{rsquare}) {
     684         my(@DataLabelPairsToAnalyze, $DataLabel1, $DataLabel2);
     685         if (@{$OptionsInfo{SpecifiedDataLabelPairs}}) {
     686           # Make sure both data field labels exist...
     687           my($DataFieldIndex);
     688           for ($DataFieldIndex = 0; (($DataFieldIndex + 1) < @{$OptionsInfo{SpecifiedDataLabelPairs}}); $DataFieldIndex += 2 ) {
     689             $DataLabel1 = $OptionsInfo{SpecifiedDataLabelPairs}[$DataFieldIndex];
     690             $DataLabel2 = $OptionsInfo{SpecifiedDataLabelPairs}[$DataFieldIndex + 1];
     691             if (exists($SDFilesInfo{AllDataLabelsMap}[$Index]{$DataLabel1}) && exists($SDFilesInfo{AllDataLabelsMap}[$Index]{$DataLabel2})) {
     692               push @DataLabelPairsToAnalyze, ($DataLabel1, $DataLabel2);
     693             }
     694           }
     695         }
     696         elsif ($OptionsInfo{AllDataLabelPairs}) {
     697           for $DataLabel1 (@{$SDFilesInfo{AllDataLabels}[$Index]}) {
     698             for $DataLabel2 (@{$SDFilesInfo{AllDataLabels}[$Index]}) {
     699               push @DataLabelPairsToAnalyze, ($DataLabel1, $DataLabel2);
     700             }
     701           }
     702         }
     703         else {
     704           for $DataLabel1 (@{$SDFilesInfo{CommonDataLabels}[$Index]}) {
     705             for $DataLabel2 (@{$SDFilesInfo{CommonDataLabels}[$Index]}) {
     706               push @DataLabelPairsToAnalyze, ($DataLabel1, $DataLabel2);
     707             }
     708           }
     709         }
     710         if (@DataLabelPairsToAnalyze) {
     711           if (@DataLabelPairsToAnalyze % 2) {
     712             warn "Warning: Ignoring file $SDFile: Invalid number  values specified using \"--datafieldpairs\" option: It must contain even number of valid values.\n";
     713             $SDFilesInfo{FileOkay}[$Index] = 0;
     714             next FILELIST;
     715           }
     716           else {
     717             for ($DataFieldIndex = 0; $DataFieldIndex < @DataLabelPairsToAnalyze; $DataFieldIndex += 2) {
     718               push @{$SDFilesInfo{DataLabelPairs1ToAnalyze}[$Index]}, $DataLabelPairsToAnalyze[$DataFieldIndex];
     719               push @{$SDFilesInfo{DataLabelPairs2ToAnalyze}[$Index]}, $DataLabelPairsToAnalyze[$DataFieldIndex + 1];
     720             }
     721             # Set up unique data field labe map as well...
     722             for $DataLabel (@DataLabelPairsToAnalyze) {
     723               if (!exists $UniqueDataLabelsToAnalyzeMap{$DataLabel}) {
     724                 $UniqueDataLabelsToAnalyzeMap{$DataLabel} = $DataLabel;
     725               }
     726             }
     727           }
     728         }
     729       }
     730       # Setup unique data field label array...
     731       push @{$SDFilesInfo{UniqueDataLabelsToAnalyze}[$Index]}, (sort keys %UniqueDataLabelsToAnalyzeMap);
     732     }
     733   }
     734 }
     735
     736 # Retrieve information about input SD files...
     737 sub RetrieveSDFilesInfo {
     738   my($SDFile, $Index, $FileDir, $FileExt, $FileName, $OutFile, $OutFileRoot, $OutFileExt, $CmpdCount);
     739
     740   %SDFilesInfo = ();
     741
     742   @{$SDFilesInfo{FileOkay}} = ();
     743   @{$SDFilesInfo{CmpdCount}} = ();
     744   @{$SDFilesInfo{NewTextFileRoot}} = ();
     745   @{$SDFilesInfo{NewTextFileExt}} = ();
     746
     747   @{$SDFilesInfo{AllDataFieldLabels}} = ();
     748   @{$SDFilesInfo{AllDataFieldLabelsMap}} = ();
     749   @{$SDFilesInfo{CommonDataLabels}} = ();
     750
     751   FILELIST: for $Index (0 .. $#SDFilesList) {
     752     $SDFile = $SDFilesList[$Index];
     753
     754     $SDFilesInfo{FileOkay}[$Index] = 0;
     755
     756     $SDFilesInfo{CmpdCount}[$Index] = 0;
     757     $SDFilesInfo{NewTextFileRoot}[$Index] = "";
     758     $SDFilesInfo{NewTextFileExt}[$Index] = "";
     759
     760     @{$SDFilesInfo{AllDataLabels}[$Index]} = ();
     761     %{$SDFilesInfo{AllDataLabelsMap}[$Index]} = ();
     762     @{$SDFilesInfo{CommonDataLabels}[$Index]} = ();
     763
     764     if (!(-e $SDFile)) {
     765       warn "Warning: Ignoring file $SDFile: It doesn't exist\n";
     766       next FILELIST;
     767     }
     768     if (!CheckFileType($SDFile, "sd sdf")) {
     769       warn "Warning: Ignoring file $SDFile: It's not a SD file\n";
     770       next FILELIST;
     771     }
     772
     773     # Generate appropriate name for the new text files...
     774     $FileDir = ""; $FileName = ""; $FileExt = "";
     775     ($FileDir, $FileName, $FileExt) = ParseFileName($SDFile);
     776     $OutFileExt = "csv";
     777     if ($Options{outdelim} =~ /^tab$/i) {
     778       $OutFileExt = "tsv";
     779     }
     780     if ($Options{root} && (@SDFilesList == 1)) {
     781       my ($RootFileDir, $RootFileName, $RootFileExt) = ParseFileName($Options{root});
     782       if ($RootFileName && $RootFileExt) {
     783         $FileName = $RootFileName;
     784       }
     785       else {
     786         $FileName = $Options{root};
     787       }
     788       $OutFileRoot = $FileName;
     789     }
     790     else {
     791       $OutFileRoot = $FileName;
     792     }
     793     $OutFile = $OutFileRoot . $OptionsInfo{FileNameMode} . ".$OutFileExt";
     794
     795     if (!$OptionsInfo{Overwrite}) {
     796       if (-e $OutFile) {
     797         warn "Warning: Ignoring file $SDFile: The file $OutFile already exists\n";
     798         next FILELIST;
     799       }
     800       if (exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{covariance}) || exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{correlation}) || exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{rsquare})) {
     801         if ($OptionsInfo{AllDataLabelPairs}) {
     802           if (exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{covariance}) && (-e "${OutFileRoot}CovarianceMatrix.${FileExt}")) {
     803             warn "Warning: Ignoring file $SDFile: The file ${OutFileRoot}Covariance.${FileExt} already exists.\n";
     804             next FILELIST;
     805           }
     806           if (exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{correlation}) && (-e "${OutFileRoot}CorrelationMatrix.${FileExt}")) {
     807             warn "Warning: Ignoring file $SDFile: The file ${OutFileRoot}CorrelationMatrix.${FileExt} already exists.\n";
     808             next FILELIST;
     809           }
     810           if (exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{rsquare}) && (-e "${OutFileRoot}RSquareMatrix.${FileExt}")) {
     811             warn "Warning: Ignoring file $SDFile: The file ${OutFileRoot}RSquareMatrix.${FileExt} already exists.\n";
     812             next FILELIST;
     813           }
     814         }
     815         else {
     816           if (-e "${OutFileRoot}ColumnPairsAnalysis.${FileExt}") {
     817             warn "Warning: Ignoring file $SDFile: The file ${OutFileRoot}ColumnPairsAnalysis.${FileExt} already exists.\n";
     818             next FILELIST;
     819           }
     820         }
     821       }
     822       if (exists($OptionsInfo{SpecifiedStatisticalFunctionsMap}{standardscores}) && (-e "${OutFileRoot}StandardScores.${FileExt}")) {
     823         warn "Warning: Ignoring file $SDFile: The file ${OutFileRoot}StandardScores.${FileExt} already exists.\n";
     824         next FILELIST;
     825       }
     826     }
     827
     828     if (!open SDFILE, "$SDFile") {
     829       warn "Warning: Ignoring file $SDFile: Couldn't open it: $! \n";
     830       next FILELIST;
     831     }
     832
     833     my($CmpdCount, $Label, $DataFieldLabelsRef, $CommonDataFieldLabelsRef, @DataFieldLabels, @CommonDataFieldLabels);
     834     $CmpdCount = 0;
     835     @DataFieldLabels = ();
     836     @CommonDataFieldLabels = ();
     837     ($CmpdCount, $DataFieldLabelsRef, $CommonDataFieldLabelsRef) = GetAllAndCommonCmpdDataHeaderLabels(\*SDFILE);
     838     push @DataFieldLabels, @{$DataFieldLabelsRef};
     839     push @CommonDataFieldLabels, @{$CommonDataFieldLabelsRef};
     840     close SDFILE;
     841
     842     $SDFilesInfo{FileOkay}[$Index] = 1;
     843     $SDFilesInfo{NewTextFileRoot}[$Index] = "$OutFileRoot";
     844     $SDFilesInfo{NewTextFileExt}[$Index] = "$OutFileExt";
     845
     846     $SDFilesInfo{CmpdCount}[$Index] = $CmpdCount;
     847     push @{$SDFilesInfo{AllDataLabels}[$Index]}, @DataFieldLabels;
     848     push @{$SDFilesInfo{CommonDataLabels}[$Index]}, @CommonDataFieldLabels;
     849     for $Label (@DataFieldLabels) {
     850       $SDFilesInfo{AllDataLabelsMap}[$Index]{$Label} = $Label;
     851     }
     852   }
     853 }
     854
     855 # Process option values...
     856 sub ProcessOptions {
     857   %OptionsInfo = ();
     858
     859   $OptionsInfo{Mode} = $Options{mode};
     860
     861   $OptionsInfo{DataFields} = defined $Options{datafields} ? $Options{datafields} : undef;
     862
     863   $OptionsInfo{DetailLevel} = $Options{detail};
     864
     865   # Setup supported statistical functions...
     866   my($SupportedFunction, @SupportedStatisticaFunctions, %SupportedStatisticaFunctionsMap);
     867
     868   %SupportedStatisticaFunctionsMap = ();
     869   @SupportedStatisticaFunctions = qw(Average AverageDeviation Correlation Count Covariance GeometricMean Frequency HarmonicMean KLargest KSmallest Kurtosis Maximum Minimum Mean Median Mode RSquare Skewness Sum SumOfSquares StandardDeviation StandardDeviationN StandardError StandardScores StandardScoresN TrimMean Variance VarianceN);
     870
     871   for $SupportedFunction (@SupportedStatisticaFunctions) {
     872     $SupportedStatisticaFunctionsMap{lc($SupportedFunction)} = $SupportedFunction;
     873   }
     874
     875   # Setup a list of functions to use for analysis...
     876   my($SpecifiedFunction);
     877
     878   %{$OptionsInfo{SpecifiedStatisticalFunctionsMap}} = ();
     879   @{$OptionsInfo{SpecifiedStatisticalFunctions}} = ();
     880
     881   # Check mode values...
     882   if ($Options{mode} =~ /^DescriptiveStatisticsBasic$/i ) {
     883     $OptionsInfo{FileNameMode} = "DescriptiveStatisticsBasic";
     884     @{$OptionsInfo{SpecifiedStatisticalFunctions}} = qw(Count Maximum Minimum Mean Median StandardDeviation StandardError Variance Sum);
     885   }
     886   elsif ($Options{mode} =~ /^DescriptiveStatisticsAll$/i ) {
     887     $OptionsInfo{FileNameMode} = "DescriptiveStatisticsAll";
     888     @{$OptionsInfo{SpecifiedStatisticalFunctions}} = qw(Count Maximum Minimum Mean GeometricMean HarmonicMean TrimMean Median Mode StandardDeviation Kurtosis Skewness StandardError Variance  RSquare Frequency  KLargest KSmallest Sum);
     889   }
     890   elsif ($Options{mode} =~ /^All$/i ) {
     891     $OptionsInfo{FileNameMode} = "AllStatistics";
     892     @{$OptionsInfo{SpecifiedStatisticalFunctions}} = @SupportedStatisticaFunctions;
     893   }
     894   else {
     895     $OptionsInfo{FileNameMode} = "SpecifiedStatistics";
     896
     897     # Comma delimited list of functions...
     898     my($Mode, @SpecifiedFunctions, @UnsupportedSpecifiedFunctions);
     899
     900     $Mode = $Options{mode};
     901     $Mode =~ s/ //g;
     902     @SpecifiedFunctions = split ",", $Mode;
     903     @UnsupportedSpecifiedFunctions = ();
     904     for $SpecifiedFunction (@SpecifiedFunctions) {
     905       if (exists($SupportedStatisticaFunctionsMap{lc($SpecifiedFunction)})) {
     906         push @{$OptionsInfo{SpecifiedStatisticalFunctions}}, $SpecifiedFunction;
     907       }
     908       else {
     909         push @UnsupportedSpecifiedFunctions, $SpecifiedFunction;
     910       }
     911     }
     912     if (@UnsupportedSpecifiedFunctions) {
     913       if (@UnsupportedSpecifiedFunctions > 1) {
     914         warn "Error: The values specified - ", JoinWords(\@UnsupportedSpecifiedFunctions, ", ", 0)," - for option \"-m --mode\" are not valid.\n";
     915       }
     916       else {
     917         warn "Error: The value specified, @UnsupportedSpecifiedFunctions , for option \"-m --mode\" is not valid.\n";
     918       }
     919       die "Allowed values:", JoinWords(\@SupportedStatisticaFunctions, ", ", 0), "\n";
     920     }
     921   }
     922
     923   FUNCTION: for $SpecifiedFunction (@{$OptionsInfo{SpecifiedStatisticalFunctions}}) {
     924     if (exists $OptionsInfo{SpecifiedStatisticalFunctionsMap}{lc($SpecifiedFunction)} ) {
     925       next FUNCTION;
     926     }
     927     $OptionsInfo{SpecifiedStatisticalFunctionsMap}{lc($SpecifiedFunction)} = $SupportedStatisticaFunctionsMap{lc($SpecifiedFunction)};
     928   }
     929
     930   # Setup delimiter and quotes...
     931   $OptionsInfo{OutDelim} = ($Options{outdelim} =~ /tab/i ) ? "\t" : (($Options{outdelim} =~ /semicolon/i) ? "\;" : "\,");
     932   $OptionsInfo{OutQuote} = ($Options{quote} =~ /yes/i ) ? 1 : 0;
     933
     934   $OptionsInfo{Overwrite} = defined $Options{overwrite} ? $Options{overwrite} : undef;
     935   $OptionsInfo{Root} = defined $Options{root} ? $Options{root} : undef;
     936
     937   # Setup miscellaneous options...
     938   $OptionsInfo{CheckData} = $Options{fast} ? 0 : 1;
     939   $OptionsInfo{Precision} = $Options{precision};
     940
     941   $OptionsInfo{KLargest} = $Options{klargest};
     942   $OptionsInfo{KSmallest} = $Options{ksmallest};
     943
     944   $OptionsInfo{TrimFraction} = $Options{trimfraction};
     945
     946   # Setup frequency bin values...
     947   $OptionsInfo{NumOfBins} = 10;
     948   @{$OptionsInfo{BinRange}} = ();
     949   if ($Options{frequencybins} =~ /\,/) {
     950     my($BinValue, @SpecifiedBinRange);
     951     @SpecifiedBinRange = split /\,/,  $Options{frequencybins};
     952     if (@SpecifiedBinRange < 2) {
     953       die "Error: The value specified, $Options{frequencybins}, for option \"--frequencybins\" is not valid: Must contain at least two values. \n";
     954     }
     955     for $BinValue (@SpecifiedBinRange) {
     956       if (!IsNumerical($BinValue)) {
     957         die "Error: The value specified, $Options{frequencybins}, for option \"--frequencybins\" is not valid: Contains non numeric values. \n";
     958       }
     959     }
     960     my($Index1, $Index2);
     961     for $Index1 (0 .. $#SpecifiedBinRange) {
     962       for $Index2 (($Index1 + 1) .. $#SpecifiedBinRange) {
     963         if ($SpecifiedBinRange[$Index1] >= $SpecifiedBinRange[$Index2]) {
     964           die "Error: The value specified, $Options{frequencybins}, for option \"--frequencybins\" is not valid: Must contain values in ascending order. \n";
     965         }
     966       }
     967     }
     968     push @{$OptionsInfo{BinRange}}, @SpecifiedBinRange;
     969   }
     970   else {
     971     $OptionsInfo{NumOfBins} = $Options{frequencybins};
     972     if (!IsPositiveInteger($OptionsInfo{NumOfBins})) {
     973       die "Error: The value specified, $Options{frequencybins}, for option \"--frequencybins\" is not valid. Allowed values: positive integer or \"number,number,[number]...\". \n";
     974     }
     975   }
     976
     977   # Setup specified data field labels...
     978   @{$OptionsInfo{SpecifiedDataLabels}} = ();
     979   if (defined $Options{datafields} && $Options{datafields} !~ /^(All|Common)$/i ) {
     980     my(@SpecifiedValues) = split ",", $Options{datafields};
     981     push @{$OptionsInfo{SpecifiedDataLabels}}, @SpecifiedValues;
     982   }
     983   @{$OptionsInfo{SpecifiedDataLabelPairs}} = ();
     984   $OptionsInfo{AllDataLabelPairs} = (defined($Options{datafieldpairs}) && $Options{datafieldpairs} =~ /^AllPairs$/i) ? 1 : 0;
     985   $OptionsInfo{CommonDataLabelPairs} = (defined($Options{datafieldpairs}) && $Options{datafieldpairs} =~ /^CommonPairs$/i) ? 1 : 0;
     986   if (defined($Options{datafieldpairs}) && !$OptionsInfo{AllDataLabelPairs} && !$OptionsInfo{CommonDataLabelPairs}) {
     987     my(@SpecifiedValues) = split ",", $Options{datafieldpairs};
     988     if (@SpecifiedValues % 2) {
     989       die "Error: Invalid number of values specified using \"--datafieldpairs\" option: It must contain even number of values.\n";
     990     }
     991     push @{$OptionsInfo{SpecifiedDataLabelPairs}}, @SpecifiedValues;
     992   }
     993
     994 }
     995
     996 # Setup script usage  and retrieve command line arguments specified using various options...
     997 sub SetupScriptUsage {
     998
     999   # Retrieve all the options...
    1000   %Options = ();
    1001   $Options{detail} = 0;
    1002   $Options{datafields} = "Common";
    1003   $Options{datafieldpairs} = "CommonPairs";
    1004   $Options{frequencybins} = 10;
    1005   $Options{klargest} = 2;
    1006   $Options{ksmallest} = 2;
    1007   $Options{mode} = "DescriptiveStatisticsBasic";
    1008   $Options{outdelim} = "comma";
    1009   $Options{precision} = 2;
    1010   $Options{quote} = "yes";
    1011   $Options{trimfraction} = 0.1;
    1012
    1013   if (!GetOptions(\%Options, "datafields=s", "datafieldpairs=s", "detail|d=i", "frequencybins=s", "fast|f", "help|h", "klargest=i", "ksmallest=i", "mode|m=s", "outdelim=s", "overwrite|o", "precision|p=i", "quote|q=s", "root|r=s", "trimfraction=f", "workingdir|w=s")) {
    1014     die "\nTo get a list of valid options and their values, use \"$ScriptName -h\" or\n\"perl -S $ScriptName -h\" command and try again...\n";
    1015   }
    1016   if ($Options{workingdir}) {
    1017     if (! -d $Options{workingdir}) {
    1018       die "Error: The value specified, $Options{workingdir}, for option \"-w --workingdir\" is not a directory name.\n";
    1019     }
    1020     chdir $Options{workingdir} or die "Error: Couldn't chdir $Options{workingdir}: $! \n";
    1021   }
    1022   if (!IsInteger($Options{detail})) {
    1023     die "Error: The value specified, $Options{detail}, for option \"-d --detail\" is not valid. Allowed values: >= 0\n";
    1024   }
    1025   if ($Options{outdelim} !~ /^(comma|semicolon|tab)$/i) {
    1026     die "Error: The value specified, $Options{outdelim}, for option \"--outdelim\" is not valid. Allowed values: comma, tab, or semicolon\n";
    1027   }
    1028   if ($Options{quote} !~ /^(yes|no)$/i) {
    1029     die "Error: The value specified, $Options{quote}, for option \"-q --quote\" is not valid. Allowed values: yes or no\n";
    1030   }
    1031   if (!IsPositiveInteger($Options{precision})) {
    1032     die "Error: The value specified, $Options{precision}, for option \"-p --precision\" is not valid. Allowed values: > 0 \n";
    1033   }
    1034   if (!IsPositiveInteger($Options{klargest})) {
    1035     die "Error: The value specified, $Options{klargest}, for option \"--klargest\" is not valid. Allowed values: > 0 \n";
    1036   }
    1037   if (!IsPositiveInteger($Options{ksmallest})) {
    1038     die "Error: The value specified, $Options{ksmallest}, for option \"--ksmallest\" is not valid. Allowed values: > 0 \n";
    1039   }
    1040   if (IsFloat($Options{trimfraction})) {
    1041     if ($Options{trimfraction} <= 0 || $Options{trimfraction} >= 1.0) {
    1042       die "Error: The value specified, $Options{trimfraction}, for option \"--trimfraction\" is not valid. Allowed values: > 0 and < 1.0\n";
    1043     }
    1044   }
    1045   else {
    1046     die "Error: The value specified, $Options{trimfraction}, for option \"--trimfraction\" is not valid. Allowed values: > 0 and < 1.0\n";
    1047   }
    1048 }
    1049