#!/usr/local/bin/perl -w use strict; =head1 Do these transforms on input html: tidy (indent, remove extra styling) retain only body innerhtml fix urls to be relative =cut use File::Basename; my $outdir = shift @ARGV; die "dir '$outdir' does not exist or is not a directory" unless -d $outdir; for my $f (@ARGV) { open(HTML , "tidy -indent -clean -ashtml -quiet -wrap 1000 < $f |") || die "can't tidy: $!"; my @lines = ; close HTML; my $html = join('', @lines); die "nothing in $f after tidy" if $html =~ m/^\s*$/s; # remove all but body innerhtml $html =~ s/.*]*>//is; $html =~ s,.*,,is; die "nothing in $f after body innerhtml" if $html =~ m/^\s*$/s; # TEMPORARY # remove all the surrounding crap if ($html =~ m,.*\n(.*)]*>[\s\n]+

(.*),si) { $html = $2; my $match = $1; # find matching close td if ($html =~ m,(.*?)\n$match(.*)$,si) { $html = $1; my $rest = $2; if ($rest =~ m,(,si) { # print STDERR "preserving form: '$1' out of '$rest'"; $html .= "\n$1"; } else { die "in $f did not preserve form" if $rest =~ m/form/si; } } else { die "in $f no closing '$match'"; } } else { die "didn't match leading crap in $f"; } die "nothing in $f after menu removal" if $html =~ m/^\s*$/s; # remove leading and trailing white $html =~ s,^[\n\s]*\n,,; die "nothing in $f after leading white removal" if $html =~ m/^\s*$/s; $html =~ s,(

|\ |[\n\s])+$,\n,; die "nothing in $f after trailing white removal" if $html =~ m/^\s*$/s; # $html .= '

' if $html =~ m/

[^<]+$/s; # we almost certainly removed a

$html .= "

\n" unless $html =~ m/p>\n*$/; # remove common indent $html =~ m/^([\s]*)/; my $indent = $1; # print STDERR "removing indent '$indent'\n"; $html =~ s,^$indent,,mg; # relativize urls $html =~ s,http:[^\'\"]*/([^\'\"]*),$1,g; # collapse empty lines $html =~ s,\n\s*\n,\n,g; # remove class attributes added by -clean $html =~ s, class=".*?",,g; if ($outdir) { my $outfile = "$outdir/" . basename($f); open(OUT, ">$outfile") || die "can't open $outfile for write: $!"; print OUT $html; close(OUT); print STDERR "wrote cleaned content to $outfile\n"; } else { print $html; } }