normalizeSpace

#!/usr/bin/env perl -w
#
# normalizeSpace: Normalize whitespace (and optionally some other things).
# 2007-10: Written by Steven J. DeRose.
#
use strict;
use Getopt::Long;
use Encode;

use alogging;

our %metadata = (
    'title'        => "normalizeSpace",
    'description'  => "Normalize whitespace (and some other things).",
    'rightsHolder' => "Steven J. DeRose",
    'creator'      => "http://viaf.org/viaf/50334488",
    'type'         => "http://purl.org/dc/dcmitype/Software",
    'language'     => "Perl 5.18",
    'created'      => "2006-10",
    'modified'     => "2020-11-30",
    'publisher'    => "http://github.com/sderose",
    'license'      => "https://creativecommons.org/licenses/by-sa/3.0/"
);
our $VERSION_DATE = $metadata{'modified'};


=pod

=head1 Usage

normalizeSpace [options] file

Reduce runs of whitespace to a single space, and drop leading and trailing
whitespace. By default, this never includes line-end characters (CR and LF),
however, see I<--evenNewlines>.
Optionally, also normalize quotes, dashes, controls, privateUse characters,
backspace, hardspace, etc.

With I<--space ALL>, all Unicode "Separator" characters
count as spaces except for CR and LF. These include regular space, TAB,
NO-BREAK SPACE, en, em, thin, and other width spaces (U+20xx), etc.
To include CR and LF as well, set I<--evenNewlines>.

Runs of spaces are squeezed to single space (this cannot be turned off yet).
leading and trailing spaces are also stripped, but those can be turned off
with I<--no-leadStrip> and I<--no-trailStrip>.

=head1 Options

(prefix 'no' to negate where applicable)

=over

=item * B<--backspace>

Add BSP (U+0008) to the list of characters counting as spaces.

=item * B<--control>

Turn each control character (U+0000 to U+001F and U+0080 to U+009F) to
a space. This happens before I<--space>, and does affect CR and LF.

B<Note>: If the "upper" ("C1") control
characters are used for graphical characters, as in the Windows "CP1252"
character set (but not Unicode or most other encodings),
this option will still turn them to spaces.
If you have this problem, you can turn off I<--control>,
translate your data out of CP1252 (e.g., with C<iconv>),
or specify I<--iencoding cp1252>.

=item * B<--dash>

Turn all dash character sequences to a single hyphen, except for
soft hyphens (U+00AD) and hyphenation points (U+2027) which are deleted,
em (clause) dashes (U+2014) which remain;
and 2 hyphens in a row, which are changed to em dash.

=item * B<--evenNewlines>
Add CR (u+000D) and LF (U+000A) to the list of characters counting as spaces.
See also I<--control>, since CR and LF are also control characters.

=item * B<--iencoding> I<e>

Specify character encoding for input. Default: C<utf8>

=item * B<--hardspace>

Add NO-BREAK SPACE (U+00A0) to the list of characters counting as spaces.

=item * B<--ilineends> I<t>

Assume Unix, Dos, or Mac line-breaks for input.

=item * B<--leadStrip>

Strip leading spaces (default; use "--no-leadStrip" to turn off such stripping).

=item * B<--listEncodings>

Show all available character encodings, then exit.

=item * B<--privateUse>

Turn all Unicode private-use characters to regular spaces.
Namely: [\x{E000}-\x{F8FF}\x{F0000}-\x{FFFFD}\x{100000}-\x{10FFFD}].

=item * B<--quiet> or B<-q>

Fewer messages.

=item * B<--quote>

Turn each Unicode initial (open) or final (close) quotation mark to '"'.
Apostrophe and backquote are not changed.

=item * B<--space> I<choice>

Turn some or all runs of Unicode Separator characters to a single space,
and also (by default) removes space from start (I<--leadStrip>) and end (I<--trailStrip>)
of lines.

Note that none of the choices affect CR or LF. To do so, you must also set
I<--evenLineends>. If you want XML attribute-style normalization, use
choice "XML", which is the same as
    --space STAB --lead --trail

The choices available are:
    STAB   -- This is the default, and affects space and tab.
    NONE   -- don't touch spaces
    SPACE  -- just the nominal SPACE character (U+20)
    LATIN1 -- all the Unicode Separator characters below 0x100.
    ALL    -- all the Unicode Separator characters

Note: You may want to delete these instead, because they are
too narrow to make a visual space:
    U+2009     THIN SPACE
    U+200A     HAIR SPACE
    U+200B     ZERO WIDTH SPACE
    U+202F     NARROW NO-BREAK SPACE

In addition, delete a leading and/or trailing space that remains after that.

See also I<--evenNewlines>, I<--controls>.

=item * B<--test>

Run a small self-test.

=item * B<--trailStrip> OR B<--tailStrip>

Strip trailing spaces (default; use "--no-trailStrip" to turn off such stripping).

=item * B<--unicode>

Synonym for I<--iencoding utf8>.

=item * B<--verbose> or B<-v>

Add more messages (repeatable).

=item * B<--version>

Show version info and exit.

=back


=head1 Known Bugs and Limitations

Seems to delete newlines when it shouldn't. Working on it.

Regex specials like \\p{Dash_Punctuation}, \\p{Initial_Punctuation}, and
\\p{Final_Punctuation} do not seem to work. Have switched to actual Unicode
character lists. That also seems to have problems. But it should still work
for normal \\s spaces.


=head1 To do

* Implement --quote for full range of Unicode quote characters.
* Perhaps recognize escaped characters like C<countChars> can.
* Perhaps add option to turn off squeezing of space runs, and/or expand TABs.


=head1 Related commands

My C<Tokenizer.pm> library provides more extensive normalization capabilities.

My C<nukeNonXmlChars> does some similar cleanups.


=head1 History

=over

=item * 2007-10: Written by Steven J. DeRose.

=item * 2012-09-07: bash-->Perl. Unicode support.
Add --control, --dash, --privateUse, --quote, --space.

=item * 2015-06-10: Get rid of \p{} in regexes; use actual Unicode char lists.

=item * 2020-10-15: Add --evenNewlines so we don't nuke CR and LF by default.
Refactor. Add --test.

=item * 2024-03-09: Improve list of space chars. Enumerate choices for
--space option. Add --leadStrip, --trailStrip, --backspace, --hardspace.

=back


=head1 Rights

Copyright 2007-10 by Steven J. DeRose. This work is licensed under a
Creative Commons Attribution-Share Alike 3.0 Unported License.
For further information on this license, see
L<https://creativecommons.org/licenses/by-sa/3.0>.

For the most recent version, see L<http://www.derose.net/steve/utilities> or
L<https://github.com/sderose>.

=cut


###############################################################################
# Pre-compile regexes, esp. for Unicode phenomena
# Character lists like these can be extract from Unicode via my 'ord' script.
#
my $controlToKillString = "\x00-\x1F\x80-\x9F";
my $controlToKillExpr = qr/([$controlToKillString])/;

my $privateToKillString =
    "\x{E000}-\x{F8FF}\x{F0000}-\x{FFFFD}\x{100000}-\x{10FFFD}";
my $privateToKillExpr = qr/([$privateToKillString])/;

###############################################################################
# Dashes
#
my $dashToKillString = "" .
    "\x{00AD}" .    # "SOFT HYPHEN"
    "\x{2027}" .    # "HYPHENATION POINT"
    "";
my $dashToKillExpr = qr/([$dashToKillString])/;

my $dashString = "" .
    "\x{002D}" .    # "HYPHEN-MINUS"        ### MUST BE FIRST FOR REGEX
    "\x{058A}" .    # "ARMENIAN HYPHEN"
    "\x{1806}" .    # "MONGOLIAN TODO SOFT HYPHEN"
    "\x{1B60}" .    # "BALINESE PAMENENG (line-breaking hyphen)"
    "\x{2010}" .    # "HYPHEN"
    "\x{2011}" .    # "NON-BREAKING HYPHEN"
    "\x{2012}" .    # "FIGURE DASH"
    "\x{2013}" .    # "EN DASH"
    ### NOT THIS ONE! "\x{2014}" .    # "EM DASH"
    "\x{2043}" .    # "HYPHEN BULLET"
    "\x{2053}" .    # "SWUNG DASH"
    # "\x{21E0}" .    # "LEFTWARDS DASHED ARROW"
    # "\x{21E1}" .    # "UPWARDS DASHED ARROW"
    # "\x{21E2}" .    # "RIGHTWARDS DASHED ARROW"
    # "\x{21E3}" .    # "DOWNWARDS DASHED ARROW"
    "\x{229D}" .    # "CIRCLED DASH"
    "\x{2448}" .    # "OCR DASH"
    # "\x{2504}" .    # "BOX DRAWINGS LIGHT TRIPLE DASH HORIZONTAL"
    # "\x{2505}" .    # "BOX DRAWINGS HEAVY TRIPLE DASH HORIZONTAL"
    # "\x{2508}" .    # "BOX DRAWINGS LIGHT QUADRUPLE DASH HORIZONTAL"
    # "\x{2509}" .    # "BOX DRAWINGS HEAVY QUADRUPLE DASH HORIZONTAL"
    # "\x{254C}" .    # "BOX DRAWINGS LIGHT DOUBLE DASH HORIZONTAL"
    # "\x{254D}" .    # "BOX DRAWINGS HEAVY DOUBLE DASH HORIZONTAL"
    "\x{2E17}" .    # "DOUBLE OBLIQUE HYPHEN"
    "\x{2E1A}" .    # "HYPHEN WITH DIAERESIS"
    "\x{301C}" .    # "WAVE DASH"
    "\x{3030}" .    # "WAVY DASH"
    "\x{30A0}" .    # "KATAKANA-HIRAGANA DOUBLE HYPHEN"
    "\x{FE49}" .    # "DASHED OVERLINE"
    "\x{FE4D}" .    # "DASHED LOW LINE"
    "\x{FE58}" .    # "SMALL EM DASH"
    "\x{FE63}" .    # "SMALL HYPHEN-MINUS"
    "\x{FF0D}" .    # "FULLWIDTH HYPHEN-MINUS"
    "";
my $dashExpr = qr/([$dashString])/;


###############################################################################
# Spaces
#
my $spaceBackSpace =
    chr(0x00008);  # BACKSPACE

my $spaceLineends =
    chr(0x0000A) .  # LINE FEED
    chr(0x0000D) .  # CARRIAGE RETURN
    "";

my $spaceSPACE =
    chr(0x00020) .  # SPACE
    "";

my $spaceHard =
    chr(0x000A0) .  # NO-BREAK SPACE
    "";

my $spaceSTAB =
    chr(0x00009) .  # CHARACTER TABULATION
    #chr(0x0000A) .  # LINE FEED
    #chr(0x0000D) .  # CARRIAGE RETURN
    chr(0x00020) .  # SPACE
    "";

my $spaceLATIN1 =
    chr(0x00009) .  # CHARACTER TABULATION
    #chr(0x0000A) .  # LINE FEED
    chr(0x0000B) .  # LINE TABULATION
    chr(0x0000C) .  # FORM FEED
    #chr(0x0000D) .  # CARRIAGE RETURN
    chr(0x00020) .  # SPACE
    chr(0x000A0) .  # NO-BREAK SPACE
    chr(0x00089) .  # CHARACTER TABULATION WITH JUSTIFICATION (WARNING: CP1252 per mille)
    "";

my $spaceALL =
    chr(0x00009) .  # CHARACTER TABULATION
    #chr(0x0000A) .  # LINE FEED
    chr(0x0000B) .  # LINE TABULATION
    chr(0x0000C) .  # FORM FEED
    #chr(0x0000D) .  # CARRIAGE RETURN
    chr(0x00020) .  # SPACE
    chr(0x00089) .  # CHARACTER TABULATION WITH JUSTIFICATION (WARNING: CP1252 per mille)
    #chr(0x000A0) .  # NO-BREAK SPACE
    chr(0x01680) .  # OGHAM SPACE MARK
    chr(0x0180E) .  # MONGOLIAN VOWEL SEPARATOR
    chr(0x02000) .  # EN QUAD
    chr(0x02001) .  # EM QUAD
    chr(0x02002) .  # EN SPACE
    chr(0x02003) .  # EM SPACE
    chr(0x02004) .  # THREE-PER-EM SPACE
    chr(0x02005) .  # FOUR-PER-EM SPACE
    chr(0x02006) .  # SIX-PER-EM SPACE
    chr(0x02007) .  # FIGURE SPACE
    chr(0x02008) .  # PUNCTUATION SPACE
    chr(0x02009) .  # THIN SPACE
    chr(0x0200A) .  # HAIR SPACE
    chr(0x02028) .  # LINE SEPARATOR
    chr(0x02029) .  # PARAGRAPH SEPARATOR
    chr(0x0200B) .  # ZERO WIDTH SPACE
    chr(0x0202F) .  # NARROW NO-BREAK SPACE
    chr(0x0205F) .  # MEDIUM MATHEMATICAL SPACE
    chr(0x02060) .  # WORD JOINER
    chr(0x03000) .  # IDEOGRAPHIC SPACE
    chr(0x0303F) .  # IDEOGRAPHIC HALF FILL SPACE
    chr(0x0FeFF) .  # ZERO WIDTH NO-BREAK SPACE (aka BOM)
    "";


###############################################################################
# Quotes
#
my $quoteString = "'\"";   ### FIX


###############################################################################
# Options
#
my $backspace     = 0;
my $control       = 0;
my $dash          = 0;
my $evenNewlines  = 0;
my $hardspace     = 0;
my $iencoding     = "utf8";
my $ilineends     = "U";
my $leadStrip     = 1;
my $trailStrip    = 1;
my $privateUse    = 0;
my $quiet         = 0;
my $quote         = 0;
my $space         = "XML";
my $test          = 0;
my $verbose       = 0;

my %getoptHash = (
    "backspace!"              => \$backspace,
    "control!"                => \$control,
    "dash!"                   => \$dash,
    "evenNewlines!"           => \$evenNewlines,
    "hardspace!"              => \$hardspace,
    "h|help"                  => sub { system "perldoc $0"; exit; },
    "iencoding=s"             => \$iencoding,
    "ilineends=s"             => \$ilineends,
    "leadStrip!"              => \$leadStrip,
    "listEncodings"    => sub {
        warn "\nEncodings available:\n";
        for my $k (Encode->encodings(":all")) {
            warn "    $k\n";
        }
        exit;
    },
    "privateUse!"             => \$privateUse,
    "quiet|q!"                => \$quiet,
    "quote!"                  => \$quote,
    "space=s"                 => \$space,
    "test!"                   => \$test,
    "trailStrip|tailStrip!"   => \$trailStrip,
    "unicode!"                => sub { $iencoding = "utf8"; },
    "v+"                      => \$verbose,
    "version"                 => sub {
        die "Version of $VERSION_DATE, by Steven J. DeRose.\n";
    },
    );
Getopt::Long::Configure ("ignore_case");
GetOptions(%getoptHash) || die "Bad options.\n";

if ($verbose) { alogging::setLogOption("verbose", $verbose) }

$ilineends = uc(substr($ilineends."U",0,1));
if    ($ilineends eq "M") { $/ = chr(13); }
elsif ($ilineends eq "D") { $/ = chr(13).chr(10); }
else { }

if ($iencoding) {
    print "";
    warn "\n";
    binmode(STDOUT, ":encoding($iencoding)");
    binmode(STDERR, ":encoding($iencoding)");
}

my $spaceString = "";
if ($evenNewlines) { $spaceString .= $spaceLineends; }
if ($backspace) { $spaceString .= $spaceBackSpace; }
if ($hardspace) { $spaceString .= $spaceHard; }

$space = uc($space);
if ($space eq "NONE") { $spaceString .= ""; }
elsif ($space eq "SPACE") { $spaceString .= $spaceSPACE; }
elsif ($space eq "STAB") { $spaceString .= $spaceSTAB; }
elsif ($space eq "LATIN1") { $spaceString .= $spaceLATIN1; }
elsif ($space eq "ALL") { $spaceString .= $spaceALL; }
elsif ($space eq "XML") { $spaceString .= $spaceSTAB . $spaceLineends; }
else { die "Bad --space value '$space'.\n"; }

my $spaceExpr = qr/([$spaceString])/;

if ($verbose) {
    warn("Chars for control: " . $controlToKillString . "\n");
    warn("Chars for private: " . $privateToKillString . "\n");
    warn("Chars for dash:    " . $dashString . "\n");
    warn("Chars for space:   " . $spaceString . "\n");
    warn("Chars for quote:   " . $quoteString . "\n");
}
($space =~ m/^(NONE|SPACE|STAB|LATIN1|ALL|XML)$/) || die
    "Unknown value '$space' for --space. See --help.\n";


###############################################################################
#
sub normalize {
    my ($rec) = @_;
    if ($control) {
        $rec =~ s/$controlToKillExpr/ /g;
    }
    if ($privateUse) {
        $rec =~ s/$privateToKillExpr/ /g;
    }
    if ($dash) {
        $rec =~ s/$dashToKillExpr//g;        # soft hyphen
        $rec =~ s/--/\x{2014}/g;             # make real em dashes
        $rec =~ s/$dashExpr/-/g;             # unify rest of dashes
    }
    if ($quote) {
        $rec =~ s/\p{Initial_Punctuation}/"/g;
        $rec =~ s/\p{Final_Punctuation}/"/g;
    }

    if ($spaceString) { $rec =~ s/$spaceExpr+/ /g; }

    if ($leadStrip) { $rec =~ s/^$spaceExpr+//; }
    if ($trailStrip) { $rec =~ s/$spaceExpr+$//; }
    return $rec;
}

sub doTest {
    print("Testing...\n");
    printf("  privateUse %d, dash %d, quote %d, space %d, evenNewlines %d, \n",
        $privateUse, $dash, $quote, $space, $evenNewlines);
    my $sample = "Here's a   bunch \t\t\t of w\xA0e\x{2003}i\xA0\xA0r\xA0\xA0d-looking\nstuff.";
    printf("    #%s#\n", $sample);
    my $result = normalize($sample);
    printf("    #%s#\n", $result);
    my $visible = $result;
    print("And with literal ASCII spaces as U+2423:\n");
    $visible =~ s/ /\x{2423}/g;
    printf("    #%s#\n", $visible);
}


###############################################################################
# Main
#
if ($test) {
    doTest();
    exit;
}

if (scalar(@ARGV)<=0) {
    push @ARGV, "-";
    if (-t STDIN && !$quiet) { warn "Waiting on STDIN...\n"; }
}

my $recnum = my $filenum = 0;
while (my $file = shift) {
    open(my $fh, "<$file") || die "Failed to open input file '$file'.\n";
    $filenum++;
    if ($iencoding) {
        binmode($fh, ":encoding($iencoding)");
        print "";
        binmode(STDOUT, ":encoding($iencoding)");
    }
    while (my $rec = <$fh>) {
        $recnum++;
        chomp $rec;
        my $buf = normalize($rec);
        print "$buf\n";
    }
    close($fh);
}

($verbose) && warn
    "normalizeSpace: Done, $filenum files, $recnum records processed.\n";
exit;