Merge branch '4.5.11'

Conflicts:
	Shorewall/Perl/Shorewall/Config.pm
	Shorewall/Perl/Shorewall/Rules.pm

Signed-off-by: Tom Eastep <teastep@shorewall.net>

Shorewall/Macros/macro.A_AllowICMPs

@@ -9,7 +9,7 @@
 #ACTION		SOURCE	DEST	PROTO	DEST	SOURCE	RATE	USER/
 #					PORT(S)	PORT(S)	LIMIT	GROUP
 
-COMMENT Needed ICMP types
+?COMMENT Needed ICMP types
 
 A_ACCEPT       -	-	icmp	fragmentation-needed
 A_ACCEPT       -	-	icmp	time-exceeded

Shorewall/Macros/macro.A_DropDNSrep

@@ -9,6 +9,6 @@
 #ACTION	SOURCE	DEST	PROTO	DEST	SOURCE	RATE	USER/
 #				PORT(S)	PORT(S)	LIMIT	GROUP
 
-COMMENT Late DNS Replies
+?COMMENT Late DNS Replies
 
 A_DROP	-	-	udp	-	53

Shorewall/Macros/macro.A_DropUPnP

@@ -9,6 +9,6 @@
 #ACTION	SOURCE	DEST	PROTO	DEST	SOURCE	RATE	USER/
 #				PORT(S)	PORT(S)	LIMIT	GROUP
 
-COMMENT UPnP
+?COMMENT UPnP
 
 A_DROP	-	-	udp	1900

Shorewall/Macros/macro.AllowICMPs

@@ -9,7 +9,7 @@
 #ACTION	SOURCE	DEST	PROTO	DEST	SOURCE	RATE	USER/
 #				PORT(S)	PORT(S)	LIMIT	GROUP
 
-COMMENT Needed ICMP types
+?COMMENT Needed ICMP types
 
 DEFAULT ACCEPT
 PARAM	-	-	icmp	fragmentation-needed

Shorewall/Macros/macro.DropDNSrep

@@ -9,7 +9,7 @@
 #ACTION	SOURCE	DEST	PROTO	DEST	SOURCE	RATE	USER/
 #				PORT(S)	PORT(S)	LIMIT	GROUP
 
-COMMENT Late DNS Replies
+?COMMENT Late DNS Replies
 
 DEFAULT DROP
 PARAM	-	-	udp	-	53

Shorewall/Macros/macro.DropUPnP

@@ -9,7 +9,7 @@
 #ACTION	SOURCE	DEST	PROTO	DEST	SOURCE	RATE	USER/
 #				PORT(S)	PORT(S)	LIMIT	GROUP
 
-COMMENT UPnP
+?COMMENT UPnP
 
 DEFAULT DROP
 PARAM	-	-	udp	1900

Shorewall/Perl/Shorewall/Compiler.pm

@@ -541,8 +541,8 @@ EOF
 #
 sub compiler {
 
-    my ( $scriptfilename, $directory, $verbosity, $timestamp , $debug, $chains , $log , $log_verbosity, $preview, $confess , $update , $annotate , $convert, $config_path, $shorewallrc                      , $shorewallrc1 ) =
+    my ( $scriptfilename, $directory, $verbosity, $timestamp , $debug, $chains , $log , $log_verbosity, $preview, $confess , $update , $annotate , $convert, $config_path, $shorewallrc                      , $shorewallrc1 , $directives ) =
-       ( '',              '',         -1,          '',          0,      '',       '',   -1,             0,        0,         0,        0,        , 0       , ''          , '/usr/share/shorewall/shorewallrc', '' );
+       ( '',              '',         -1,          '',          0,      '',       '',   -1,             0,        0,         0,        0,        , 0       , ''          , '/usr/share/shorewall/shorewallrc', ''            , 0 );
 
     $export = 0;
     $test   = 0;
@@ -579,6 +579,7 @@ sub compiler {
 		  update        => { store => \$update,        validate=> \&validate_boolean    } ,
 		  convert       => { store => \$convert,       validate=> \&validate_boolean    } ,
 		  annotate      => { store => \$annotate,      validate=> \&validate_boolean    } ,
+		  directives    => { store => \$directives,    validate=> \&validate_boolean    } ,
 		  config_path   => { store => \$config_path } ,
 		  shorewallrc   => { store => \$shorewallrc } ,
 		  shorewallrc1  => { store => \$shorewallrc1 } ,
@@ -617,7 +618,7 @@ sub compiler {
     #
     #                      S H O R E W A L L . C O N F  A N D  C A P A B I L I T I E S
     #
-    get_configuration( $export , $update , $annotate );
+    get_configuration( $export , $update , $annotate , $directives );
     #
     # Create a temp file to hold the script
     #

Shorewall/Perl/Shorewall/Config.pm

@@ -131,6 +131,7 @@ our %EXPORT_TAGS = ( internal => [ qw( create_temp_script
 				       run_user_exit1
 				       run_user_exit2
 				       generate_aux_config
+				       format_warning
 				       process_comment
 				       no_comment
 				       macro_comment
@@ -199,7 +200,7 @@ our %EXPORT_TAGS = ( internal => [ qw( create_temp_script
 
 Exporter::export_ok_tags('internal');
 
-our $VERSION = '4.5.11-Beta1';
+our $VERSION = '4.5_11';
 
 #
 # describe the current command, it's present progressive, and it's completion.
@@ -496,9 +497,11 @@ my  $max_format;             # Max format value
 our $comment;                # Current COMMENT
 my  @comments;
 my  $comments_allowed;
-my $warningcount;
+my  $warningcount;
+my  $warningcount1;
+my  $warningcount2;
 
-my $shorewall_dir;           # Shorewall Directory; if non-empty, search here first for files.
+my  $shorewall_dir;          # Shorewall Directory; if non-empty, search here first for files.
 
 our $debug;                  # Global debugging flag
 my  $confess;                # If true, use Carp to report errors with stack trace.
@@ -511,9 +514,9 @@ our $Product;                # $product with initial cap.
 
 our $sillyname;              # Name of temporary filter chains for testing capabilities
 our $sillyname1;
-my $iptables;                # Path to iptables/ip6tables
+my  $iptables;               # Path to iptables/ip6tables
-my $tc;                      # Path to tc
+my  $tc;                     # Path to tc
-my $ip;                      # Path to ip
+my  $ip;                     # Path to ip
 
 my $shell;                   # Type of shell that processed the params file
 
@@ -618,9 +621,11 @@ sub initialize( $;$$) {
     #
     # Contents of last COMMENT line.
     #
-    $comment      = '';
+    $comment       = '';
-    @comments     = ();
+    @comments      = ();
-    $warningcount = 0;
+    $warningcount  = 0;
+    $warningcount1 = 0;
+    $warningcount2 = 0;
     #
     # Misc Globals
     #
@@ -632,7 +637,7 @@ sub initialize( $;$$) {
 		    EXPORT     => 0,
 		    KLUDGEFREE => '',
 		    STATEMATCH => '-m state --state',
-		    VERSION    => "4.5.8-Beta2",
+		    VERSION    => "4.5.11-RC1",
 		    CAPVERSION => 40509 ,
 		  );
     #
@@ -1923,11 +1928,19 @@ sub split_line($$) {
     &split_line1( @_, {} );
 }
 
+#
+# Generate a FORMAT warning
+#
+sub format_warning() {
+    warning_message "'FORMAT' is deprecated in favor of '?FORMAT' - consider running '$product update -D'" unless $warningcount2++; 
+}    
+
 #
 # Process a COMMENT line (in $currentline)
 #
 sub process_comment() {
     if ( have_capability( 'COMMENTS' ) ) {
+	warning_message "'COMMENT' is deprecated in favor of '?COMMENT' - consider running '$product update -D'" unless $warningcount1++; 
 	( $comment = $currentline ) =~ s/^\s*COMMENT\s*//;
 	$comment =~ s/\s*$//;
     } else {
@@ -2546,14 +2559,14 @@ EOF
 # The following two functions allow module clients to nest opens. This happens frequently
 # in the Rules module.
 #
-sub push_open( $;$ ) {
+sub push_open( $;$$ ) {
-    my ( $file, $max ) = @_;
+    my ( $file, $max , $ca) = @_;
     push @includestack, [ $currentfile, $currentfilename, $currentlinenumber, $ifstack, $file_format, $max_format ] if $currentfile;
     my @a = @includestack;
     push @openstack, \@a;
     @includestack = ();
     $currentfile = undef;
-    open_file( $file , $max, $comments_allowed );
+    open_file( $file , $max, $comments_allowed || $ca );
 }
 
 sub pop_open() {
@@ -4672,15 +4685,71 @@ sub export_params() {
     }
 }
 
+#
+# Walk the CONFIG_PATH converting FORMAT and COMMENT lines to compiler directives
+#
+sub convert_to_directives() {
+    my $sharedir = $shorewallrc{SHAREDIR};
+    #
+    # Make a copy of @config_path so that the for-loop below doesn't clobber that list
+    #
+    my @path = @config_path;
+
+    $sharedir =~ s|/+$||;
+
+    my $dirtest = qr|^$sharedir/+shorewall6?(?:/.*)?$|;
+
+    progress_message3 "Converting 'FORMAT' and 'COMMENT' lines to compiler directives...";
+
+    for my $dir ( @path ) {
+	unless ( $dir =~ /$dirtest/ || ! -w $dir ) {
+	    $dir =~ s|/+$||;
+
+	    opendir( my $dirhandle, $dir ) || fatal_error "Cannot open directory $dir for reading:$!";
+
+	    while ( my $file = readdir( $dirhandle ) ) {
+		unless ( $file eq 'capabilities' || $file =~ /\.bak$/ ) {
+		    $file = "$dir/$file";
+		
+		    if ( -f $file && -w _ ) {
+			#
+			# writeable regular file
+			#
+			my $result = system << "EOF";
+perl -pi.bak -e '/^\\s*FORMAT\\s*/ && s/FORMAT/?FORMAT/;
+                 if ( /^\\s*COMMENT\\s+/ ) {
+                     s/COMMENT/?COMMENT/;
+                 } elsif ( /^\\s*COMMENT\\s*\$/ ) {
+                     s/COMMENT/?COMMENT/;
+                 }' $file
+EOF
+			if ( $result == 0 ) {
+			    if ( system( "diff -q $file ${file}.bak > /dev/null" ) ) {
+				progress_message3 "   File $file updated - old file renamed ${file}.bak";
+			    } elsif ( ! unlink "${file}.bak" ) {
+
+			    }
+			} else {
+			    warning_message ("Unable to update file ${file}.bak:$!" );
+			}
+		    }
+		}
+	    }
+
+	    closedir $dirhandle;
+	}
+    }
+}
+
 #
 # - Process the params file
 # - Read the shorewall.conf file
 # - Read the capabilities file, if any
 # - establish global hashes %params, %config , %globals and %capabilities
 #
-sub get_configuration( $$$ ) {
+sub get_configuration( $$$$ ) {
 
-    my ( $export, $update, $annotate ) = @_;
+    my ( $export, $update, $annotate, $directives ) = @_;
 
     $globals{EXPORT} = $export;
 
@@ -5207,7 +5276,10 @@ sub get_configuration( $$$ ) {
     while ( my ($var, $val ) = each %renamed ) {
 	$variables{$var} = $config{$val};
     }
+
+    convert_to_directives if $directives;
 }
+
 #
 # The values of the options in @propagateconfig are copied to the script file in OPTION=<value> format.
 #

Shorewall/Perl/Shorewall/Raw.pm

@@ -205,6 +205,7 @@ sub process_format( $ ) {
     my $format = shift;
 
     fatal_error q(FORMAT must be '1', '2' or '3') unless $format =~ /^[123]$/;
+    format_warning;
 
     $file_format = $format;
 }

Shorewall/Perl/Shorewall/Rules.pm

@@ -1472,7 +1472,7 @@ sub process_actions() {
     $targets{$_} = new_action( $_ , ACTION + BUILTIN, 1, 0 ) for @builtins;
 
     for my $file ( qw/actions.std actions/ ) {
-	open_file( $file, 2, 1 );
+	open_file( $file, 2 );
 
 	while ( read_a_line( NORMAL_READ ) ) {
 	    my ( $action, $options ) = split_line 'action file' , { action => 0, options => 1 };
@@ -1552,7 +1552,7 @@ sub process_action($) {
 
 	progress_message2 "$doing $actionfile for chain $chainref->{name}...";
 
-	push_open $actionfile, 2;
+	push_open $actionfile, 2, 1;
 
 	my $oldparms = push_action_params( $chainref, $param, $level, $tag );
 
@@ -1584,6 +1584,7 @@ sub process_action($) {
 	    }
 
 	    if ( $target eq 'FORMAT' ) {
+		format_warning;
 		fatal_error "FORMAT must be 1 or 2" unless $source =~ /^[12]$/;
 		$file_format = $source;
 		next;
@@ -1688,6 +1689,7 @@ sub process_macro ($$$$$$$$$$$$$$$$$$$) {
 	}
 
 	if ( $mtarget eq 'FORMAT' ) {
+	    format_warning;
 	    fatal_error "Invalid FORMAT ($msource)" unless $msource =~ /^[12]$/;
 	    $file_format = $msource;
 	    next;
@@ -1796,7 +1798,7 @@ sub process_inline ($$$$$$$$$$$$$$$$$$$$) {
 
     progress_message "..Expanding inline action $inlinefile...";
 
-    push_open $inlinefile;
+    push_open $inlinefile, 2;
 
     while ( read_a_line( NORMAL_READ ) ) {
 	my  ( $mtarget,
@@ -1828,7 +1830,7 @@ sub process_inline ($$$$$$$$$$$$$$$$$$$$) {
 	}
 
 	if ( $mtarget eq 'FORMAT' ) {
-	    fatal_error "FORMAT must be 2" unless $source ne '2';
+	    fatal_error "FORMAT must be 2" unless $msource eq '2';
 	    next;
 	}
 

Shorewall/Perl/Shorewall/Tc.pm

@@ -225,6 +225,7 @@ sub process_tc_rule( ) {
     }
 
     if ( $originalmark eq 'FORMAT' ) {
+	format_warning;
 	if ( $source =~ /^([12])$/ ) {
 	    $file_format = $1;
 	    return;
@@ -1877,7 +1878,7 @@ sub process_tcinterfaces() {
 #
 sub process_tcpri() {
     my $fn  = find_file 'tcinterfaces';
-    my $fn1 = open_file 'tcpri';
+    my $fn1 = open_file 'tcpri', 1,1;
 
     if ( $fn1 ) {
 	first_entry

Shorewall/Perl/Shorewall/Zones.pm

@@ -1008,6 +1008,7 @@ sub process_interface( $$ ) {
     }
 
     if ( $zone eq 'FORMAT' ) {
+	format_warning;
 	if ( $originalinterface =~ /^([12])$/ ) {
 	    $file_format = $1;
 	    return;

Shorewall/Perl/compiler.pl

@@ -67,6 +67,7 @@ sub usage( $ ) {
     [ --annotate ]
     [ --update ]
     [ --convert ]
+    [ --directives ]
     [ --shorewallrc=<pathname> ]
     [ --shorewallrc1=<pathname> ]
     [ --config_path=<path-list> ]
@@ -94,6 +95,7 @@ my $preview       = 0;
 my $annotate      = 0;
 my $update        = 0;
 my $convert       = 0;
+my $directives    = 0;
 my $config_path   = '';
 my $shorewallrc   = '';
 my $shorewallrc1  = '';
@@ -124,6 +126,8 @@ my $result = GetOptions('h'               => \$help,
 			'confess'         => \$confess,
 			'a'               => \$annotate,
 			'annotate'        => \$annotate,
+			'directives'      => \$directives,
+			'D'               => \$directives,
 			'u'               => \$update,
 			'update'          => \$update,
 			'convert'         => \$convert,
@@ -151,6 +155,7 @@ compiler( script          => $ARGV[0] || '',
 	  update          => $update,
 	  convert         => $convert,
 	  annotate        => $annotate,
+	  directives      => $directives,
 	  config_path     => $config_path,
 	  shorewallrc     => $shorewallrc,
 	  shorewallrc1    => $shorewallrc1,

Shorewall/lib.cli-std

@@ -426,6 +426,7 @@ compiler() {
     [ -n "$g_update" ] && options="$options --update"
     [ -n "$g_convert" ] && options="$options --convert"
     [ -n "$g_annotate" ] && options="$options --annotate"
+    [ -n "$g_directives" ] && options="$options --directives"
 
     if [ -n "$PERL" ]; then
 	if [ ! -x "$PERL" ]; then
@@ -734,10 +735,6 @@ check_command() {
 			    g_confess=Yes
 			    option=${option#T}
 			    ;;
-			a*)
-			    g_annotate=Yes
-			    option=${option#a}
-			    ;;
 			*)
 			    usage 1
 			    ;;
@@ -826,6 +823,10 @@ update_command() {
 			    g_convert=Yes
 			    option=${option#b}
 			    ;;
+			D*)
+			    g_directives=Yes
+			    option=${option#D}
+			    ;;
 			*)
 			    usage 1
 			    ;;
@@ -1668,7 +1669,7 @@ usage() # $1 = exit status
     echo "   status"
     echo "   stop"
     echo "   try <directory> [ <timeout> ]"
-    echo "   update [ -a ] [ -b ] [ -r ] [ -T ] [ <directory> ]"
+    echo "   update [ -a ] [ -b ] [ -r ] [ -T ] [ -D ] [ <directory> ]"
     echo "   version [ -a ]"
     echo
     exit $1

Shorewall/manpages/shorewall.xml

@@ -674,6 +674,8 @@
 
       <arg><option>-a</option></arg>
 
+      <arg><option>-D</option></arg>
+
       <arg><replaceable>directory</replaceable></arg>
     </cmdsynopsis>
 
@@ -1723,6 +1725,13 @@
           <ulink url="shorewall-hosts.html">shorewall-hosts</ulink> (5). The
           unmodified files are saved with a .bak suffix.</para>
 
+          <para>The <option>-D</option> option was added in Shorewall 4.5.11.
+          When this option is specified, the compiler will walk through the
+          directories in the CONFIG_PATH replacing FORMAT and COMMENT entries
+          to compiler directives (e.g., ?FORMAT and ?COMMENT. When a file is
+          updated, the original is saved in a .bak file in the same
+          directory.</para>
+
           <para>For a description of the other options, see the <emphasis
           role="bold">check</emphasis> command above.</para>
         </listitem>

Shorewall6/action.A_AllowICMPs

@@ -8,7 +8,7 @@
 ###############################################################################
 #TARGET		SOURCE		DEST	PROTO		DEST
 #							PORT(S)
-COMMENT Needed ICMP types (RFC4890)
+?COMMENT Needed ICMP types (RFC4890)
 
 A_ACCEPT	-		-	ipv6-icmp	destination-unreachable
 A_ACCEPT	-		-	ipv6-icmp	packet-too-big

Shorewall6/action.AllowICMPs

@@ -12,7 +12,7 @@
 ?FORMAT 2
 DEFAULTS ACCEPT
 
-COMMENT Needed ICMP types (RFC4890)
+?COMMENT Needed ICMP types (RFC4890)
 
 $1	-		-	ipv6-icmp	destination-unreachable
 $1	-		-	ipv6-icmp	packet-too-big

Shorewall6/manpages/shorewall6.xml

@@ -591,6 +591,8 @@
 
       <arg><option>-a</option></arg>
 
+      <arg><option>-D</option></arg>
+
       <arg><replaceable>directory</replaceable></arg>
     </cmdsynopsis>
 
@@ -1562,6 +1564,13 @@
           and <ulink url="shorewall6-hosts.html">shorewall6-hosts</ulink> (5).
           The unmodified files are saved with a .bak suffix.</para>
 
+          <para>The <option>-D</option> option was added in Shorewall 4.5.11.
+          When this option is specified, the compiler will walk through the
+          directories in the CONFIG_PATH replacing FORMAT and COMMENT entries
+          to compiler directives (e.g., ?FORMAT and ?COMMENT. When a file is
+          updated, the original is saved in a .bak file in the same
+          directory.</para>
+
           <para>For a description of the other options, see the <emphasis
           role="bold">check</emphasis> command above.</para>
         </listitem>

docs/upgrade_issues.xml

@@ -256,6 +256,59 @@
 
           <member><filename>tcrules</filename></member>
         </simplelist>
+
+        <para>The first instance of 'FORMAT' (without the '?') will generate
+        this warning:</para>
+
+        <simplelist>
+          <member>WARNING: FORMAT is deprecated in favor of ?FORMAT; consider
+          running 'shorewall update -D'</member>
+        </simplelist>
+
+        <para>As the warning suggests, 'shorewall[6] update -D' will convert
+        all instances of FORMAT to ?FORMAT in files on the CONFIG_PATH.</para>
+      </listitem>
+
+      <listitem>
+        <para>Also beginning with Shorewalll 4.5.11, ?COMMENT is preferred
+        over COMMENT for specifying comments to be attached to generated
+        Netfilter rules in the following files:</para>
+
+        <simplelist>
+          <member><filename>accounting</filename></member>
+
+          <member><filename>action</filename>.* files</member>
+
+          <member><filename>blrules</filename></member>
+
+          <member><filename>conntrack</filename></member>
+
+          <member><filename>macro</filename>.* files</member>
+
+          <member><filename>masq</filename></member>
+
+          <member><filename>nat</filename></member>
+
+          <member><filename>rules</filename></member>
+
+          <member><filename>secmarks</filename></member>
+
+          <member><filename>tcrules</filename></member>
+
+          <member><filename>tunnels</filename></member>
+        </simplelist>
+
+        <para>The first instance of 'COMMENT' (without the '?') will generate
+        this warning:</para>
+
+        <simplelist>
+          <member>WARNING: COMMENT is deprecated in favor of ?COMMENT;
+          consider running 'shorewall update -D'</member>
+        </simplelist>
+
+        <para>As the warning suggests, 'shorewall[6] update -D' will convert
+        all instances of COMMENT to ?COMMENT in files on the
+        CONFIG_PATH.</para>
       </listitem>
 
       <listitem>