Implement update -D

Signed-off-by: Tom Eastep <teastep@shorewall.net>

Shorewall/Perl/Shorewall/Compiler.pm

@@ -541,8 +541,8 @@ EOF
 #
 sub compiler {
 
-    my ( $scriptfilename, $directory, $verbosity, $timestamp , $debug, $chains , $log , $log_verbosity, $preview, $confess , $update , $annotate , $convert, $config_path, $shorewallrc                      , $shorewallrc1 ) =
+    my ( $scriptfilename, $directory, $verbosity, $timestamp , $debug, $chains , $log , $log_verbosity, $preview, $confess , $update , $annotate , $convert, $config_path, $shorewallrc                      , $shorewallrc1 , $directives ) =
-       ( '',              '',         -1,          '',          0,      '',       '',   -1,             0,        0,         0,        0,        , 0       , ''          , '/usr/share/shorewall/shorewallrc', '' );
+       ( '',              '',         -1,          '',          0,      '',       '',   -1,             0,        0,         0,        0,        , 0       , ''          , '/usr/share/shorewall/shorewallrc', ''            , 0 );
 
     $export = 0;
     $test   = 0;
@@ -579,6 +579,7 @@ sub compiler {
 		  update        => { store => \$update,        validate=> \&validate_boolean    } ,
 		  convert       => { store => \$convert,       validate=> \&validate_boolean    } ,
 		  annotate      => { store => \$annotate,      validate=> \&validate_boolean    } ,
+		  directives    => { store => \$directives,    validate=> \&validate_boolean    } ,
 		  config_path   => { store => \$config_path } ,
 		  shorewallrc   => { store => \$shorewallrc } ,
 		  shorewallrc1  => { store => \$shorewallrc1 } ,
@@ -617,7 +618,7 @@ sub compiler {
     #
     #                      S H O R E W A L L . C O N F  A N D  C A P A B I L I T I E S
     #
-    get_configuration( $export , $update , $annotate );
+    get_configuration( $export , $update , $annotate , $directives );
     #
     # Create a temp file to hold the script
     #

Shorewall/Perl/Shorewall/Config.pm

@@ -131,6 +131,7 @@ our %EXPORT_TAGS = ( internal => [ qw( create_temp_script
 				       run_user_exit1
 				       run_user_exit2
 				       generate_aux_config
+				       format_warning
 				       process_comment
 				       no_comment
 				       macro_comment
@@ -199,7 +200,7 @@ our %EXPORT_TAGS = ( internal => [ qw( create_temp_script
 
 Exporter::export_ok_tags('internal');
 
-our $VERSION = '4.5.11-Beta1';
+our $VERSION = '4.5_11';
 
 #
 # describe the current command, it's present progressive, and it's completion.
@@ -497,6 +498,8 @@ our $comment;                # Current COMMENT
 my  @comments;
 my  $comments_allowed;
 my $warningcount;
+my $warningcount1;
+my $warningcount2;
 
 my $shorewall_dir;           # Shorewall Directory; if non-empty, search here first for files.
 
@@ -618,9 +621,11 @@ sub initialize( $;$$) {
     #
     # Contents of last COMMENT line.
     #
-    $comment      = '';
+    $comment       = '';
-    @comments     = ();
+    @comments      = ();
-    $warningcount = 0;
+    $warningcount  = 0;
+    $warningcount1 = 0;
+    $warningcount2 = 0;
     #
     # Misc Globals
     #
@@ -632,7 +637,7 @@ sub initialize( $;$$) {
 		    EXPORT     => 0,
 		    KLUDGEFREE => '',
 		    STATEMATCH => '-m state --state',
-		    VERSION    => "4.5.8-Beta2",
+		    VERSION    => "4.5.11-RC1",
 		    CAPVERSION => 40509 ,
 		  );
     #
@@ -1923,11 +1928,19 @@ sub split_line($$) {
     &split_line1( @_, {} );
 }
 
+#
+# Generate a FORMAT warning
+#
+sub format_warning() {
+    warning_message "'FORMAT' is deprecated in favor of '?FORMAT' - consider running '$product update -D'" unless $warningcount2++; 
+}    
+
 #
 # Process a COMMENT line (in $currentline)
 #
 sub process_comment() {
     if ( have_capability( 'COMMENTS' ) ) {
+	warning_message "'COMMENT' is deprecated in favor of '?COMMENT' - consider running '$product update -D'" unless $warningcount1++; 
 	( $comment = $currentline ) =~ s/^\s*COMMENT\s*//;
 	$comment =~ s/\s*$//;
     } else {
@@ -4672,15 +4685,71 @@ sub export_params() {
     }
 }
 
+#
+# Walk the CONFIG_PATH converting FORMAT and COMMENT lines to compiler directives
+#
+sub convert_to_directives() {
+    my $sharedir = $shorewallrc{SHAREDIR};
+    #
+    # Make a copy of @config_path so that the for-loop below doesn't clobber that list
+    #
+    my @path = @config_path;
+
+    $sharedir =~ s|/+$||;
+
+    my $dirtest = qr|^$sharedir/+shorewall6?(?:/.*)?$|;
+
+    progress_message3 "Converting 'FORMAT' and 'COMMENT' lines to compiler directives...";
+
+    for my $dir ( @path ) {
+	unless ( $dir =~ /$dirtest/ || ! -w $dir ) {
+	    $dir =~ s|/+$||;
+
+	    opendir( my $dirhandle, $dir ) || fatal_error "Cannot open directory $dir for reading:$!";
+
+	    while ( my $file = readdir( $dirhandle ) ) {
+		unless ( $file eq 'capabilities' || $file =~ /\.bak$/ ) {
+		    $file = "$dir/$file";
+		
+		    if ( -f $file && -w _ ) {
+			#
+			# writeable regular file
+			#
+			my $result = system << "EOF";
+perl -pi.bak -e '/^\\s*FORMAT\\s*/ && s/FORMAT/?FORMAT/;
+                 if ( /^\\s*COMMENT\\s+/ ) {
+                     s/COMMENT/?COMMENT/;
+                 } elsif ( /^\\s*COMMENT\\s*\$/ ) {
+                     s/COMMENT/?COMMENT/;
+                 }' $file
+EOF
+			if ( $result == 0 ) {
+			    if ( system( "diff -q $file ${file}.bak > /dev/null" ) ) {
+				progress_message3 "   File $file updated - old file renamed ${file}.bak";
+			    } elsif ( ! unlink "${file}.bak" ) {
+
+			    }
+			} else {
+			    warning_message ("Unable to update file ${file}.bak:$!" );
+			}
+		    }
+		}
+	    }
+
+	    closedir $dirhandle;
+	}
+    }
+}
+
 #
 # - Process the params file
 # - Read the shorewall.conf file
 # - Read the capabilities file, if any
 # - establish global hashes %params, %config , %globals and %capabilities
 #
-sub get_configuration( $$$ ) {
+sub get_configuration( $$$$ ) {
 
-    my ( $export, $update, $annotate ) = @_;
+    my ( $export, $update, $annotate, $directives ) = @_;
 
     $globals{EXPORT} = $export;
 
@@ -5207,7 +5276,10 @@ sub get_configuration( $$$ ) {
     while ( my ($var, $val ) = each %renamed ) {
 	$variables{$var} = $config{$val};
     }
+
+    convert_to_directives if $directives;
 }
+
 #
 # The values of the options in @propagateconfig are copied to the script file in OPTION=<value> format.
 #

Shorewall/Perl/Shorewall/Raw.pm

@@ -205,6 +205,7 @@ sub process_format( $ ) {
     my $format = shift;
 
     fatal_error q(FORMAT must be '1', '2' or '3') unless $format =~ /^[123]$/;
+    format_warning;
 
     $file_format = $format;
 }

Shorewall/Perl/Shorewall/Rules.pm

@@ -1584,6 +1584,7 @@ sub process_action($) {
 	    }
 
 	    if ( $target eq 'FORMAT' ) {
+		format_warning;
 		fatal_error "FORMAT must be 1 or 2" unless $source =~ /^[12]$/;
 		$file_format = $source;
 		next;
@@ -1688,6 +1689,7 @@ sub process_macro ($$$$$$$$$$$$$$$$$$$) {
 	}
 
 	if ( $mtarget eq 'FORMAT' ) {
+	    format_warning;
 	    fatal_error "Invalid FORMAT ($msource)" unless $msource =~ /^[12]$/;
 	    $file_format = $msource;
 	    next;
@@ -1828,7 +1830,7 @@ sub process_inline ($$$$$$$$$$$$$$$$$$$$) {
 	}
 
 	if ( $mtarget eq 'FORMAT' ) {
-	    fatal_error "FORMAT must be 2" unless $source ne '2';
+	    fatal_error "FORMAT must be 2" unless $msource eq '2';
 	    next;
 	}
 

Shorewall/Perl/Shorewall/Zones.pm

@@ -1008,6 +1008,7 @@ sub process_interface( $$ ) {
     }
 
     if ( $zone eq 'FORMAT' ) {
+	format_warning;
 	if ( $originalinterface =~ /^([12])$/ ) {
 	    $file_format = $1;
 	    return;

Shorewall/Perl/compiler.pl

@@ -67,6 +67,7 @@ sub usage( $ ) {
     [ --annotate ]
     [ --update ]
     [ --convert ]
+    [ --directives ]
     [ --shorewallrc=<pathname> ]
     [ --shorewallrc1=<pathname> ]
     [ --config_path=<path-list> ]
@@ -94,6 +95,7 @@ my $preview       = 0;
 my $annotate      = 0;
 my $update        = 0;
 my $convert       = 0;
+my $directives    = 0;
 my $config_path   = '';
 my $shorewallrc   = '';
 my $shorewallrc1  = '';
@@ -124,6 +126,8 @@ my $result = GetOptions('h'               => \$help,
 			'confess'         => \$confess,
 			'a'               => \$annotate,
 			'annotate'        => \$annotate,
+			'directives'      => \$directives,
+			'D'               => \$directives,
 			'u'               => \$update,
 			'update'          => \$update,
 			'convert'         => \$convert,
@@ -151,6 +155,7 @@ compiler( script          => $ARGV[0] || '',
 	  update          => $update,
 	  convert         => $convert,
 	  annotate        => $annotate,
+	  directives      => $directives,
 	  config_path     => $config_path,
 	  shorewallrc     => $shorewallrc,
 	  shorewallrc1    => $shorewallrc1,

Shorewall/lib.cli-std

@@ -426,6 +426,7 @@ compiler() {
     [ -n "$g_update" ] && options="$options --update"
     [ -n "$g_convert" ] && options="$options --convert"
     [ -n "$g_annotate" ] && options="$options --annotate"
+    [ -n "$g_directives" ] && options="$options --directives"
 
     if [ -n "$PERL" ]; then
 	if [ ! -x "$PERL" ]; then
@@ -734,10 +735,6 @@ check_command() {
 			    g_confess=Yes
 			    option=${option#T}
 			    ;;
-			a*)
-			    g_annotate=Yes
-			    option=${option#a}
-			    ;;
 			*)
 			    usage 1
 			    ;;
@@ -826,6 +823,10 @@ update_command() {
 			    g_convert=Yes
 			    option=${option#b}
 			    ;;
+			D*)
+			    g_directives=Yes
+			    option=${option#D}
+			    ;;
 			*)
 			    usage 1
 			    ;;
@@ -1668,7 +1669,7 @@ usage() # $1 = exit status
     echo "   status"
     echo "   stop"
     echo "   try <directory> [ <timeout> ]"
-    echo "   update [ -a ] [ -b ] [ -r ] [ -T ] [ <directory> ]"
+    echo "   update [ -a ] [ -b ] [ -r ] [ -T ] [ -D ] [ <directory> ]"
     echo "   version [ -a ]"
     echo
     exit $1

Shorewall/manpages/shorewall.xml

@@ -674,6 +674,8 @@
 
       <arg><option>-a</option></arg>
 
+      <arg><option>-D</option></arg>
+
       <arg><replaceable>directory</replaceable></arg>
     </cmdsynopsis>
 
@@ -1723,6 +1725,13 @@
           <ulink url="shorewall-hosts.html">shorewall-hosts</ulink> (5). The
           unmodified files are saved with a .bak suffix.</para>
 
+          <para>The <option>-D</option> option was added in Shorewall 4.5.11.
+          When this option is specified, the compiler will walk through the
+          directories in the CONFIG_PATH replacing FORMAT and COMMENT entries
+          to compiler directives (e.g., ?FORMAT and ?COMMENT. When a file is
+          updated, the original is saved in a .bak file in the same
+          directory.</para>
+
           <para>For a description of the other options, see the <emphasis
           role="bold">check</emphasis> command above.</para>
         </listitem>

Shorewall6/manpages/shorewall6.xml

@@ -591,6 +591,8 @@
 
       <arg><option>-a</option></arg>
 
+      <arg><option>-D</option></arg>
+
       <arg><replaceable>directory</replaceable></arg>
     </cmdsynopsis>
 
@@ -1562,6 +1564,13 @@
           and <ulink url="shorewall6-hosts.html">shorewall6-hosts</ulink> (5).
           The unmodified files are saved with a .bak suffix.</para>
 
+          <para>The <option>-D</option> option was added in Shorewall 4.5.11.
+          When this option is specified, the compiler will walk through the
+          directories in the CONFIG_PATH replacing FORMAT and COMMENT entries
+          to compiler directives (e.g., ?FORMAT and ?COMMENT. When a file is
+          updated, the original is saved in a .bak file in the same
+          directory.</para>
+
           <para>For a description of the other options, see the <emphasis
           role="bold">check</emphasis> command above.</para>
         </listitem>

docs/upgrade_issues.xml

@@ -35,7 +35,7 @@
 
       <holder>Thomas M. Eastep</holder>
 
-      <holder/>
+      <holder></holder>
     </copyright>
 
     <legalnotice>
@@ -163,7 +163,7 @@
           <para>?ENDIF.</para>
         </blockquote>
 
-        <para/>
+        <para></para>
       </listitem>
 
       <listitem>
@@ -256,6 +256,17 @@
 
           <member><filename>tcrules</filename></member>
         </simplelist>
+
+        <para>The first instance of 'FORMAT' (without the '?') will generate
+        this warning:</para>
+
+        <simplelist>
+          <member>WARNING: FORMAT is deprecated in favor of ?FORMAT; consider
+          running 'shorewall update -D'</member>
+        </simplelist>
+
+        <para>As the warning suggests, 'shorewall[6] update -D' will convert
+        all instances of FORMAT to ?FORMAT in files on the CONFIG_PATH.</para>
       </listitem>
 
       <listitem>
@@ -286,6 +297,18 @@
 
           <member><filename>tunnels</filename></member>
         </simplelist>
+
+        <para>The first instance of 'COMMENT' (without the '?') will generate
+        this warning:</para>
+
+        <simplelist>
+          <member>WARNING: COMMENT is deprecated in favor of ?COMMENT;
+          consider running 'shorewall update -D'</member>
+        </simplelist>
+
+        <para>As the warning suggests, 'shorewall[6] update -D' will convert
+        all instances of COMMENT to ?COMMENT in files on the
+        CONFIG_PATH.</para>
       </listitem>
     </orderedlist>
   </section>