Merge branch '5.1.12'

Shorewall-core/lib.cli

@@ -3238,8 +3238,8 @@ report_capabilities_unsorted() {
     [ -n "$RECENT_MATCH" ] && report_capability 'Recent Match "--reap" option (REAP_OPTION)' $REAP_OPTION
     report_capability "Owner Match (OWNER_MATCH)" $OWNER_MATCH
     report_capability "Owner Name Match (OWNER_NAME_MATCH)" $OWNER_NAME_MATCH
-    if [ -n "$IPSET_MATCH" ]; then
     report_capability "Ipset Match (IPSET_MATCH)" $IPSET_MATCH
+    if [ -n "$IPSET_MATCH" ]; then
 	[ -n "$OLD_IPSET_MATCH" ]     && report_capability "OLD_Ipset Match (OLD_IPSET_MATCH)"           $OLD_IPSET_MATCH
 	[ -n "$IPSET_MATCH_NOMATCH" ] && report_capability "Ipset Match Nomatch (IPSET_MATCH_NOMATCH)"   $IPSET_MATCH_NOMATCH
 	[ -n "$IPSET_MATCH_NOMATCH" ] && report_capability "Ipset Match Counters (IPSET_MATCH_COUNTERS)" $IPSET_MATCH_COUNTERS

Shorewall/Perl/Shorewall/Chains.pm

@@ -1319,12 +1319,12 @@ sub pop_match( $$ ) {
 
 sub clone_irule( $ );
 
-sub format_rule( $$;$ ) {
+sub format_rule( $$ ) {
-    my ( $chainref, $rulerefp, $suppresshdr ) = @_;
+    my ( $chainref, $rulerefp ) = @_;
 
     return $rulerefp->{cmd} if exists $rulerefp->{cmd};
 
-    my $rule = $suppresshdr ? '' : "-A $chainref->{name}";
+    my $rule = "-A $chainref->{name}";
     #
     # The code that follows can be destructive of the rule so we clone it
     #
@@ -3377,15 +3377,43 @@ sub delete_references( $ ) {
 #
 # Calculate a digest for the passed chain and store it in the {digest} member.
 #
+# First, a lightweight version of format_rule()
+#
+sub irule_to_string( $ ) {
+    my ( $ruleref ) = @_;
+
+    return $ruleref->{cmd} if exists $ruleref->{cmd};
+
+    my $string = '';
+
+    for ( grep ! ( get_opttype( $_, 0 ) & ( CONTROL | TARGET ) ), @{$ruleref->{matches}} ) {
+	my $value = $ruleref->{$_};
+	if ( reftype $value ) {
+	    $string .= "$_=" . join( ',', @$value ) . ' ';
+	} else {
+	    $string .= "$_=$value ";
+	}
+    }
+
+    if ( $ruleref->{target} ) {
+	$string .= join( ' ', " -$ruleref->{jump}", $ruleref->{target} );
+	$string .= join( '', ' ', $ruleref->{targetopts} ) if $ruleref->{targetopts};
+    }
+
+    $string .= join( '', ' -m comment --comment "', $ruleref->{comment}, '"' ) if $ruleref->{comment};
+
+    $string;
+}
+
 sub calculate_digest( $ ) {
     my $chainref = shift;
     my $rules = '';
 
     for ( @{$chainref->{rules}} ) {
 	if ( $rules ) {
-	    $rules .= ' |' . format_rule( $chainref, $_, 1 );
+	    $rules .= ' |' . irule_to_string( $_ );
 	} else {
-	    $rules = format_rule( $chainref, $_, 1 );
+	    $rules = irule_to_string( $_ );
 	}
     }
 
@@ -3857,7 +3885,10 @@ sub optimize_level8( $$$ ) {
     %renamed = ();
 
     while ( $progress ) {
-	my @chains   = ( sort { level8_compare($a, $b) } ( grep $_->{referenced} && ! $_->{builtin}, values %{$tableref} ) );
+	my @chains   = ( sort { level8_compare($a, $b) } ( grep $_->{referenced} &&
+							   @{$_->{rules}}        &&
+							   ! $_->{builtin},
+							   values %{$tableref} ) );
 	my @chains1  = @chains;
 	my $chains   = @chains;
 	my %rename;
@@ -3877,12 +3908,11 @@ sub optimize_level8( $$$ ) {
 	    # Shift the current $chainref off of @chains1
 	    #
 	    shift @chains1;
+
+	    for my $chainref1 (grep ! ( $_->{optflags} & DONT_DELETE ), @chains1 ) {
 		#
-	    # Skip empty chains
+		# Chains identical?
 		#
-	    for my $chainref1 ( @chains1 ) {
-		next unless @{$chainref1->{rules}};
-		next if $chainref1->{optflags} & DONT_DELETE;
 		if ( $chainref->{digest} eq $chainref1->{digest} ) {
 		    progress_message "  Chain $chainref1->{name} combined with $chainref->{name}";
 		    $progress = 1;
@@ -8199,19 +8229,8 @@ sub add_interface_options( $ ) {
 	# Generate a digest for each chain
 	#
 	for my $chainref ( values %input_chains, values %forward_chains ) {
-	    my $digest = '';
-
 	    assert( $chainref );
-
+	    calculate_digest( $chainref );
-	    for ( @{$chainref->{rules}} ) {
-		if ( $digest ) {
-		    $digest .= ' |' . format_rule( $chainref, $_, 1 );
-		} else {
-		    $digest = format_rule( $chainref, $_, 1 );
-		}
-	    }
-
-	    $chainref->{digest} = sha1_hex $digest;
 	}
 	#
 	# Insert jumps to the interface chains into the rules chains
@@ -8509,7 +8528,7 @@ sub save_dynamic_chains() {
 	);
 
     if ( have_capability 'IPTABLES_S' ) {
-	emit <<"EOF";
+	emithd <<"EOF";
 if chain_exists 'UPnP -t nat'; then
     $tool -t nat -S UPnP | tail -n +2 > \${VARDIR}/.UPnP
 else
@@ -8530,6 +8549,7 @@ fi
 EOF
 	if ( $config{MINIUPNPD} ) {
 	    emit << "EOF";
+
 if chain_exists 'MINIUPNPD-POSTROUTING -t nat'; then
     $tool -t nat -S MINIUPNPD-POSTROUTING | tail -n +2 > \${VARDIR}/.MINIUPNPD-POSTROUTING
 else
@@ -8538,7 +8558,7 @@ fi
 EOF
 	}
     } else {
-	emit <<"EOF";
+	emithd <<"EOF";
 if chain_exists 'UPnP -t nat'; then
     $utility -t nat | grep '^-A UPnP ' > \${VARDIR}/.UPnP
 else
@@ -8558,7 +8578,8 @@ else
 fi
 EOF
 	if ( $config{MINIUPNPD} ) {
-	    emit << "EOF";
+	    emithd << "EOF";
+
 if chain_exists 'MINIUPNPD-POSTROUTING -t nat'; then
     $utility -t nat | grep '^-A MINIUPNPD-POSTROUTING' > \${VARDIR}/.MINIUPNPD-POSTROUTING
 else
@@ -8572,7 +8593,7 @@ EOF
     emit ( 'else' );
     push_indent;
 
-emit <<"EOF";
+emithd <<"EOF";
 rm -f \${VARDIR}/.UPnP
 rm -f \${VARDIR}/.forwardUPnP
 EOF
@@ -8609,7 +8630,7 @@ sub ensure_ipsets( @ ) {
 
 	pop_indent;
 
-	emit( qq(    fi\n) );
+	emit( q(    fi) );
 
     }
 
@@ -8785,7 +8806,6 @@ sub create_load_ipsets() {
 		  '        $IPSET flush $set' ,
 		  '        $IPSET destroy $set' ,
 		  "    done" ,
-		  '',
 		);
 	} else {
 	    #
@@ -8797,7 +8817,7 @@ sub create_load_ipsets() {
 		   '    fi' );
 	};
 
-	emit( '}' );
+	emit( "}\n" );
     }
     #
     # Now generate load_ipsets()
@@ -8866,20 +8886,17 @@ sub create_load_ipsets() {
 	    ensure_ipsets( @ipsets );
 
 	    emit( 'elif [ "$COMMAND" = refresh ]; then' );   ################### Refresh Command ###################
-	    emit ( '' );
 	    ensure_ipsets( @ipsets );
-	    emit ( '' );
 	};
 
-	emit ( 'fi' ,
+	emit ( 'fi' );
-	       '' );
     } else {
 	emit 'true';
     }
 
     pop_indent;
 
-    emit '}';	    
+    emit "}\n";	    
 }
 
 #
@@ -9052,7 +9069,7 @@ sub create_netfilter_load( $ ) {
 	  "cat \${VARDIR}/.${utility}-input | \$command # Use this nonsensical form to appease SELinux",
 	  'if [ $? != 0 ]; then',
 	  qq(    fatal_error "iptables-restore Failed. Input is in \${VARDIR}/.${utility}-input"),
-	  "fi\n"
+	  'fi'
 	);
 
     pop_indent;

Shorewall/Perl/Shorewall/Compiler.pm

@@ -103,7 +103,7 @@ sub generate_script_1( $ ) {
 
     copy2( $lib, $debug ) if -f $lib;
 
-    emit <<'EOF';
+    emithd<<'EOF';
 ################################################################################
 # Functions to execute the various user exits (extension scripts)
 ################################################################################
@@ -125,7 +125,7 @@ EOF
 	emit '}';
     }
 
-    emit <<'EOF';
+    emithd <<'EOF';
 ################################################################################
 # End user exit functions
 ################################################################################
@@ -270,12 +270,11 @@ sub generate_script_2() {
 	    );
 	emit( 'chain_exists DOCKER-INGRESS   && g_dockeringress=Yes' );
 	emit( 'chain_exists DOCKER-ISOLATION && g_dockernetwork=Yes' );
-	emit( '' );
     }
 
     pop_indent;
 
-    emit "\n}\n"; # End of initialize()
+    emit "}\n"; # End of initialize()
 
     emit( '' ,
 	  '#' ,
@@ -525,7 +524,7 @@ sub generate_script_3($) {
 
     my $config_dir = $globals{CONFIGDIR};
 
-    emit<<"EOF";
+    emithd <<"EOF";
     set_state Started $config_dir
     run_restored_exit
 elif [ \$COMMAND = refresh ]; then
@@ -572,7 +571,7 @@ EOF
 	  '    run_started_exit',
 	  "fi\n" );
 
-    emit<<'EOF';
+    emithd<<'EOF';
 date > ${VARDIR}/restarted
 
 case $COMMAND in

Shorewall/Perl/Shorewall/Config.pm

@@ -189,6 +189,7 @@ our %EXPORT_TAGS = ( internal => [ qw( create_temp_script
 		                       in_hex8
 		                       in_hexp
 				       emit
+				       emithd
 				       emitstd
 				       emit_unindented
 				       save_progress_message
@@ -835,7 +836,7 @@ sub initialize( $;$$$) {
 		    TC_SCRIPT               => '',
 		    EXPORT                  => 0,
 		    KLUDGEFREE              => '',
-		    VERSION                 => "5.1.8-Beta1",
+		    VERSION                 => "5.1.12-Beta2",
 		    CAPVERSION              => 50112 ,
 		    BLACKLIST_LOG_TAG       => '',
 		    RELATED_LOG_TAG         => '',
@@ -1689,6 +1690,7 @@ sub emit {
 		$line =~ s/^\n// if $lastlineblank;
 		$line =~ s/^/$indent/gm if $indent;
 		$line =~ s/        /\t/gm;
+		$line =~ s/[ \t]+\n/\n/gm;
 		print $script "$line\n" if $script;
 		$lastlineblank = ( substr( $line, -1, 1 ) eq "\n" );
 
@@ -1709,6 +1711,15 @@ sub emit {
     }
 }
 
+#
+# Used to emit a 'here documents' string without introducing an unwanted blank line at the end
+#
+sub emithd( $ ) {
+    my ( $line ) = @_; #make writable
+    chomp $line;
+    emit $line;
+}
+
 #
 # Version of emit() that writes to standard out unconditionally
 #
@@ -1719,6 +1730,7 @@ sub emitstd {
 	    $line =~ s/^\n// if $lastlineblank;
 	    $line =~ s/^/$indent/gm if $indent;
 	    $line =~ s/        /\t/gm;
+	    $line =~ s/[ \t]+\n/\n/gm;
 	    print "$line\n";
 	    $lastlineblank = ( substr( $line, -1, 1 ) eq "\n" );
 	} else {

Shorewall/Perl/Shorewall/Misc.pm

@@ -2646,7 +2646,6 @@ EOF
 
         rm -f ${VARDIR}/proxyarp
     fi
-
 EOF
     } else {
 	emit <<'EOF';
@@ -2660,7 +2659,6 @@ EOF
 
         rm -f ${VARDIR}/proxyndp
     fi
-
 EOF
     }
 

Shorewall/Perl/Shorewall/Providers.pm

@@ -885,7 +885,8 @@ sub add_a_provider( $$ ) {
 		emit( "run_ip rule add from $address pref 20000 table $id" ,
 		      "echo \"\$IP -$family rule del from $address pref 20000> /dev/null 2>&1\" >> \${VARDIR}/undo_${table}_routing" );
 	    } else {
-		emit  ( "find_interface_addresses $physical | while read address; do",
+		emit  ( '',
+			"find_interface_addresses $physical | while read address; do",
 			"    qt \$IP -$family rule del from \$address",
 			"    run_ip rule add from \$address pref 20000 table $id",
 			"    echo \"\$IP -$family rule del from \$address pref 20000 > /dev/null 2>&1\" >> \${VARDIR}/undo_${table}_routing",
@@ -1250,7 +1251,7 @@ CEOF
 		   'if [ $COMMAND = disable ]; then',
 		   "    do_persistent_${what}_${table}",
 		   "else",
-		   "    echo 1 > \${VARDIR}/${physical}_disabled\n",
+		   "    echo 1 > \${VARDIR}/${physical}_disabled",
 		   "fi\n",
 		 );
 	}
@@ -1593,7 +1594,8 @@ sub finish_providers() {
 	}
 
 	if ( $config{USE_DEFAULT_RT} ) {
-	    emit  ( "    while qt \$IP -$family route del default table $main; do",
+	    emit  ( '',
+		    "    while qt \$IP -$family route del default table $main; do",
 		    '        true',
 		    '    done',
 		    ''
@@ -1739,7 +1741,7 @@ sub process_providers( $ ) {
 
     add_a_provider( $providers{$_}, $tcdevices ) for @providers;
 
-    emit << 'EOF';;
+    emithd << 'EOF';;
 
 #
 # Enable an optional provider
@@ -1785,12 +1787,11 @@ EOF
     pop_indent;
     pop_indent;
 
-    emit << 'EOF';;
+    emithd << 'EOF';;
         *)
             startup_error "$g_interface is not an optional provider or interface"
             ;;
     esac
-
 }
 
 #
@@ -1894,20 +1895,19 @@ sub setup_providers() {
 
 	start_providers;
 
-	setup_null_routing if $config{NULL_ROUTE_RFC1918};
+	setup_null_routing, emit '' if $config{NULL_ROUTE_RFC1918};
-
-	emit '';
 
+	if ( @providers ) {
 	    emit "start_$providers{$_}->{what}_$_" for @providers;
-
 	    emit '';
+	}
 
 	finish_providers;
 
 	emit "\nrun_ip route flush cache";
 
 	pop_indent;
-	emit "fi\n";
+	emit 'fi';
 
 	setup_route_marking if @routemarked_interfaces || @load_interfaces;
     } else {
@@ -1918,9 +1918,10 @@ sub setup_providers() {
 	if ( $pseudoproviders ) {
 	    emit '';
 	    emit "start_$providers{$_}->{what}_$_" for @providers;
+	    emit '';
 	}
 
-	emit "\nundo_routing";
+	emit "undo_routing";
 	emit "restore_default_route $config{USE_DEFAULT_RT}";
 
 	my $standard_routes = @{$providers{main}{routes}} || @{$providers{default}{routes}};
@@ -1945,7 +1946,7 @@ sub setup_providers() {
 
 	pop_indent;
 
-	emit "fi\n";
+	emit 'fi';
     }
 }
 

Shorewall/Perl/Shorewall/Proxyarp.pm

@@ -96,6 +96,7 @@ sub setup_one_proxy_arp( $$$$$$$ ) {
     }
 
     emit ( "run_ip neigh add proxy $address nud permanent dev $extphy" ,
+	   '' ,
 	   qq(progress_message "   Host $address connected to $interface added to $proto on $extphy"\n) );
 
     push @proxyarp, "$address $interface $external $haveroute";

Shorewall/manpages/shorewall.conf.xml

@@ -2134,6 +2134,14 @@ LOG:info:,bar                        net                    fw</programlisting>
               <para>Optimization category 8 - Added in Shorewall 4.4.9. When
               set, causes chains with identical rules to be collapsed into a
               single chain.</para>
+
+              <warning>
+                <para>While Optimization category 8 can significantly reduce
+                the size of the generated iptables ruleset, it can also take
+                significant system resources during compilation. If you find
+                that compilation takes an unreasonably long time, try
+                disabling this category by setting OPTIMIZE=23.</para>
+              </warning>
             </listitem>
 
             <listitem>
@@ -2206,7 +2214,8 @@ LOG:info:,bar                        net                    fw</programlisting>
 
           <para>In versions prior to 5.1.0, the default value is zero which
           disables all optimizations. Beginning with Shorewall 5.1.0, the
-          default value is All which enables all optimizations.</para>
+          default value is <emphasis role="bold">All</emphasis> which enables
+          all optimizations.</para>
         </listitem>
       </varlistentry>
 

docs/Actions.xml

@@ -81,8 +81,9 @@
     <orderedlist>
       <listitem>
         <para>Built-in Actions. These actions are known by the Shorewall code
-        itself. They are listed in the comments at the top of the file
+        itself. They were formerly listed in the comments at the top of the
-        <filename>/usr/share/shorewall/actions.std</filename>.</para>
+        file <filename>/usr/share/shorewall/actions.std</filename>. They have
+        now been replaced by Standard Actions.</para>
       </listitem>
 
       <listitem>
@@ -115,8 +116,11 @@ ACCEPT   -              -               tcp     135,139,445</programlisting>
         file to <filename class="directory">/etc/shorewall</filename> (or
         somewhere else on your CONFIG_PATH) and modify the copy.</para>
 
-        <para>Standard Actions have been largely replaced by <ulink
+        <para>You can see a list of the standard actions with a short
-        url="Macros.html">macros</ulink> .</para>
+        description of each action using the <command>shorewall show
+        actions</command> command. You can display the contents of
+        action.<replaceable>name </replaceable>by typing s<command>horewall
+        show action <replaceable>name</replaceable></command>.</para>
       </listitem>
 
       <listitem>

docs/FAQ.xml

@@ -2225,7 +2225,8 @@ Creating input Chains...
         <listitem>
           <para>Don't start Shorewall at boot time (Debian and Ubuntu users
           may simply set startup=0 in
-          <filename>/etc/default/shorewall</filename>).</para>
+          <filename>/etc/default/shorewall</filename>) or disable in systemd
+          using <command>systemctl disable shorewall.service</command>.</para>
         </listitem>
 
         <listitem>
@@ -2349,6 +2350,22 @@ gateway:~# </programlisting>
           <command>reload</command> avoids the <command>stop</command>
           part.</para>
         </listitem>
+
+        <listitem>
+          <para>Use a capabilities file:</para>
+
+          <itemizedlist>
+            <listitem>
+              <para>Run <command>shorewall show -f capabilties &gt;
+              /etc/shorewall/capabilities </command></para>
+            </listitem>
+
+            <listitem>
+              <para>Rerun that command each time you install a new kernel or a
+              new version of shorewall.</para>
+            </listitem>
+          </itemizedlist>
+        </listitem>
       </orderedlist>
     </section>
 

docs/Macros.xml

@@ -102,6 +102,14 @@ PARAM    -       -       tcp     135,139,445</programlisting>
         somewhere else on your CONFIG_PATH) and modify the copy.</para>
       </listitem>
 
+      <listitem>
+        <para>You can see a list of the Standard Macros in your version of
+        Shorewall using the <command>shorewall show macros</command> command.
+        You can see the contents of the file
+        macro.<replaceable>name</replaceable> by typing <command>shorewall
+        show macro <replaceable>name</replaceable></command>.</para>
+      </listitem>
+
       <listitem>
         <para>User-defined Macros. These macros are created by end-users. They
         are defined in macro.* files in /etc/shorewall or in another directory
@@ -796,19 +804,20 @@ bar:debug</programlisting>
 
     <orderedlist>
       <listitem>
-        <para>You can not associate an Extension Script with a macro <ulink
+        <para>Embedded Perl is <ulink url="???">much more useful in an
-        url="Actions.html#Extension">the way that you can with an
+        action</ulink> than it is in a macro. So if you need access to
-        Action</ulink>. So if you need access to iptables features not
+        iptables features not directly supported by Shorewall then you should
-        directly supported by Shorewall then you must use an action.</para>
+        use an action.</para>
       </listitem>
 
       <listitem>
-        <para>Macros are expanded in-line while each action is its own chain.
+        <para>Macros are expanded in-line while each action (that doesn't
-        So if there are a lot of rules involved in your new action/macro then
+        specify the inline option) is its own chain. So if there are a lot of
-        it is generally better to use an action than a macro. Only the packets
+        rules involved in your new action/macro then it is generally better to
-        selected when you invoke the action are directed to the corresponding
+        use an action than a macro. Only the packets selected when you invoke
-        chain. On the other hand, if there are only one or two rules involved
+        the action are directed to the corresponding chain. On the other hand,
-        in what you want to do then a macro is more efficient.</para>
+        if there are only one or two rules involved in what you want to do
+        then a macro is more efficient.</para>
       </listitem>
     </orderedlist>