Reflect changes in tools repository

Signed-off-by: Tom Eastep <teastep@shorewall.net>

Shorewall-init/shorewall-init

@@ -25,6 +25,7 @@
 #
 ###############################################################################
 # set the STATEDIR variable
+
 setstatedir() {
     local statedir
     if [ -f ${CONFDIR}/${PRODUCT}/vardir ]; then
@@ -59,12 +60,18 @@ else
     exit 1
 fi
 
-# Initialize the firewall
-shorewall_start () {
+# Initialize the firewalls
+
+shorewall_init_start () {
     local PRODUCT
     local STATEDIR
 
     printf "Initializing \"Shorewall-based firewalls\": "
+
+    if [ -n "$SAVE_IPSETS" -a -f "$SAVE_IPSETS" ]; then
+	ipset -R < "$SAVE_IPSETS"
+    fi
+
     for PRODUCT in $PRODUCTS; do
 	if setstatedir; then
 	    #
@@ -78,19 +85,17 @@ shorewall_start () {
 	fi
     done
 
-    if [ -n "$SAVE_IPSETS" -a -f "$SAVE_IPSETS" ]; then
-	ipset -R < "$SAVE_IPSETS"
-    fi
-
     return 0
 }
 
-# Clear the firewall
-shorewall_stop () {
+# Clear the firewalls
+
+shorewall_init_stop () {
     local PRODUCT
     local STATEDIR
 
     printf "Clearing \"Shorewall-based firewalls\": "
+
     for PRODUCT in $PRODUCTS; do
 	if setstatedir; then
 	    #
@@ -118,10 +123,10 @@ shorewall_stop () {
 
 case "$1" in
     start)
-	shorewall_start
+	shorewall_init_start
 	;;
     stop)
-	shorewall_stop
+	shorewall_init_stop
 	;;
     *)
 	echo "Usage: $0 {start|stop}"

Shorewall/Perl/Shorewall/Chains.pm

@@ -726,6 +726,7 @@ our %opttype = ( rule          => CONTROL,
 		 'icmpv6-type' => UNIQUE,
 
 		 comment       => CONTROL,
+		 digest        => CONTROL,
 
 		 policy        => MATCH,
 		 state         => EXCLUSIVE,
@@ -3521,6 +3522,33 @@ sub irule_to_string( $ ) {
     $string;
 }
 
+#
+# This one omits the comment
+#
+sub irule_to_string1( $ ) {
+    my ( $ruleref ) = @_;
+
+    return $ruleref->{cmd} if exists $ruleref->{cmd};
+
+    my $string = '';
+
+    for ( grep ! ( get_opttype( $_, 0 ) & ( CONTROL | TARGET ) ), @{$ruleref->{matches}}) {
+	my $value = $ruleref->{$_};
+	if ( reftype $value ) {
+	    $string .= "$_=" . join( ',', @$value ) . ' ';
+	} else {
+	    $string .= "$_=$value ";
+	}
+    }
+
+    if ( $ruleref->{target} ) {
+	$string .= join( ' ', " -$ruleref->{jump}", $ruleref->{target} );
+	$string .= join( '', ' ', $ruleref->{targetopts} ) if $ruleref->{targetopts};
+    }
+
+    $string;
+}
+
 sub calculate_digest( $ ) {
     my $chainref = shift;
     my $rules = '';
@@ -4190,10 +4218,10 @@ sub get_multi_sports( $ ) {
 }
 
 #
-# Return an array of keys for the passed rule. 'dport', 'comment', and 'origin' are omitted;
+# Return an array of keys for the passed rule. 'dport', 'comment', 'origin' and 'digest' are omitted;
 #
 sub get_keys( $ ) {
-    my %skip = ( dport => 1, comment => 1, origin => 1 );
+    my %skip = ( dport => 1, comment => 1, origin => 1, digest => 1 );
 
     sort grep ! $skip{$_},  keys %{$_[0]};
 }
@@ -4374,64 +4402,54 @@ sub delete_duplicates {
     my @rules;
     my $chainref  = shift;
     my $lastrule  = @_;
-    my $baseref   = pop;
     my $ruleref;
     my %skip = ( comment => 1, origin => 1 );
 
+    for ( @_ ) {
+	$_->{digest} = sha1_hex irule_to_string1( $_ );
+    }
+
+    my $baseref   = pop;
+
     while ( @_ ) {
 	my $docheck;
 	my $duplicate = 0;
 
 	if ( $baseref->{mode} == CAT_MODE && $baseref->{target} ) {
 	    my $ports1;
-	    my @keys1    = sort( grep ! $skip{$_}, keys( %$baseref ) );
+	    my $bad_key;
 	    my $rulenum  = @_;
 	    my $adjacent = 1;
-		
-	    {
-	      RULE:
+	    my $digest   = $baseref->{digest};
 
-		while ( --$rulenum >= 0 ) {
-		    $ruleref = $_[$rulenum];
+	    for ( grep ! $skip{$_}, keys( %$baseref ) ) {
+		$bad_key = 1, last if $bad_match{$_};
+	    }
 
-		    last unless $ruleref->{mode} == CAT_MODE;
+	    while ( --$rulenum >= 0 ) {
+		$ruleref = $_[$rulenum];
 
-		    my @keys2 = sort(grep ! $skip{$_}, keys( %$ruleref ) );
+		last unless $ruleref->{mode} == CAT_MODE;
 
-		    next unless @keys1 == @keys2 ;
+		next unless $digest eq $ruleref->{digest};
 
-		    my $keynum = 0;
-
-		    if ( $adjacent > 0 ) {
-			#
-			# There are no non-duplicate rules between this rule and the base rule
-			#
-			for my $key (  @keys1 ) {
-			    next RULE unless $key eq $keys2[$keynum++];
-			    next RULE unless compare_values( $baseref->{$key}, $ruleref->{$key} );
-			}
-		    } else {
-			#
-			# There are non-duplicate rules between this rule and the base rule
-			#
-			for my $key ( @keys1 ) {
-			    next RULE unless $key eq $keys2[$keynum++];
-			    next RULE unless compare_values( $baseref->{$key}, $ruleref->{$key} );
-			    last RULE if $bad_match{$key};
-			}
-		    }
+		unless ( $adjacent > 0 ) {
 		    #
-		    # This rule is a duplicate
+		    # There are non-duplicate rules between this rule and the base rule
 		    #
-		    $duplicate = 1;
-		    #
-		    # Increment $adjacent so that the continue block won't set it to zero
-		    #
-		    $adjacent++;
-
-		} continue {
-		    $adjacent--;
+		    last if $bad_key;
 		}
+		#
+		# This rule is a duplicate
+		#
+		$duplicate = 1;
+		#
+		# Increment $adjacent so that the continue block won't set it to zero
+		#
+		$adjacent++;
+
+	    } continue {
+		$adjacent--;
 	    }
 	}
 
@@ -4468,10 +4486,10 @@ sub get_conntrack( $ ) {
 }
 
 #
-# Return an array of keys for the passed rule. 'conntrack',  'comment' & 'origin' are omitted;
+# Return an array of keys for the passed rule. 'conntrack',  'comment', 'origin' and 'digest' are omitted;
 #
 sub get_keys1( $ ) {
-    my %skip = ( comment => 1, origin => 1 , 'conntrack --ctstate' => 1 );
+    my %skip = ( comment => 1, origin => 1 , digest => 1, 'conntrack --ctstate' => 1 );
 
     sort grep ! $skip{$_},  keys %{$_[0]};
 }
@@ -9065,10 +9083,14 @@ sub create_load_ipsets() {
 	    # Requires V5 or later
 	    #
 	    emit( '' ,
-		  "    for set in \$(\$IPSET save | grep '$select' | cut -d' ' -f2); do" ,
-		  '        $IPSET flush $set' ,
-		  '        $IPSET destroy $set' ,
-		  "    done" ,
+		  '    if  [ -f ${VARDIR}/ipsets.save ]; then' ,
+		  '        while read verb set rest; do' ,
+		  '            if [ $verb = create ]; then' ,
+		  '                $IPSET flush $set' ,
+		  '                $IPSET destroy $set' ,
+		  '            fi' ,
+		  '        done < ${VARDIR}/ipsets.save' ,
+		  '    fi',
 		);
 	} else {
 	    #

Shorewall/Perl/Shorewall/Zones.pm

@@ -2028,7 +2028,7 @@ sub verify_required_interfaces( $ ) {
 
 	push_indent;
 
-	emit( 'start|reload|restore)' );
+	emit( 'start|reload|restore|enable)' );
 
 	push_indent;
 

Shorewall/Perl/lib.runtime

@@ -1113,7 +1113,7 @@ interface_is_usable() # $1 = interface
     status=0
 
     if [ "$1" != lo ]; then
-	if interface_is_up $1 && [ "$(find_first_interface_address_if_any $1)" != :: ] && [ -z "$($IP -$g_family link list dev $1 2> /dev/null | fgrep 'state DOWN')" ]; then
+	if interface_is_up $1 && [ "$(find_first_interface_address_if_any $1)" != :: ]; then
 	    if [ "$COMMAND" != enable ]; then
 		[ ! -f ${VARDIR}/${1}_disabled ] && run_isusable_exit $1
 		status=$?

Shorewall/Shorewall-targetname

@@ -1 +1 @@
-5.2.4.1
+5.2.4.5

Shorewall/lib.cli-std

@@ -386,6 +386,7 @@ uptodate() {
 
     [ -n "${find}" ] || return 1
     [ -h "${find}" ] && busybox=Yes
+    find="${find} -L"
 
     for dir in $g_shorewalldir $(split $CONFIG_PATH); do
 	if [ -n "${busybox}" ]; then

Shorewall/manpages/shorewall-interfaces.xml

@@ -70,8 +70,7 @@
           in this column.</para>
 
           <para>If the interface serves multiple zones that will be defined in
-          the <ulink
-          url="shorewall-hosts.html">shorewall-hosts</ulink>(5)
+          the <ulink url="shorewall-hosts.html">shorewall-hosts</ulink>(5)
           file, you should place "-" in this column.</para>
 
           <para>If there are multiple interfaces to the same zone, you must
@@ -109,8 +108,8 @@ loc     eth2            -</programlisting>
           <para>When using Shorewall versions before 4.1.4, care must be
           exercised when using wildcards where there is another zone that uses
           a matching specific interface. See <ulink
-          url="shorewall-nesting.html">shorewall-nesting</ulink>(5)
-          for a discussion of this problem.</para>
+          url="shorewall-nesting.html">shorewall-nesting</ulink>(5) for a
+          discussion of this problem.</para>
 
           <para>Shorewall allows '+' as an interface name, but that usage is
           deprecated. A better approach is to specify
@@ -370,8 +369,7 @@ loc     eth2            -</programlisting>
                 firewall through this interface and whether the source address
                 and/or destination address is to be compared against the
                 ipset-based dynamic blacklist (DYNAMIC_BLACKLIST=ipset... in
-                <ulink
-                url="shorewall.conf.html">shorewall.conf(5)</ulink>).
+                <ulink url="shorewall.conf.html">shorewall.conf(5)</ulink>).
                 The default is determine by the setting of
                 DYNAMIC_BLACKLIST:</para>
 
@@ -459,8 +457,8 @@ loc     eth2            -</programlisting>
 
                   <listitem>
                     <para>the interface is a <ulink
-                    url="../SimpleBridge.html">simple bridge</ulink> with a DHCP
-                    server on one port and DHCP clients on another
+                    url="../SimpleBridge.html">simple bridge</ulink> with a
+                    DHCP server on one port and DHCP clients on another
                     port.</para>
 
                     <note>
@@ -585,8 +583,8 @@ loc     eth2            -</programlisting>
               <listitem>
                 <para>Connection requests from this interface are compared
                 against the contents of <ulink
-                url="shorewall-maclist.html">shorewall-maclist</ulink>(5).
-                If this option is specified, the interface must be an Ethernet
+                url="shorewall-maclist.html">shorewall-maclist</ulink>(5). If
+                this option is specified, the interface must be an Ethernet
                 NIC and must be up before Shorewall is started.</para>
               </listitem>
             </varlistentry>
@@ -650,8 +648,8 @@ loc     eth2            -</programlisting>
 
                 <para>Smurfs will be optionally logged based on the setting of
                 SMURF_LOG_LEVEL in <ulink
-                url="shorewall.conf.html">shorewall.conf</ulink>(5).
-                After logging, the packets are dropped.</para>
+                url="shorewall.conf.html">shorewall.conf</ulink>(5). After
+                logging, the packets are dropped.</para>
               </listitem>
             </varlistentry>
 
@@ -659,6 +657,11 @@ loc     eth2            -</programlisting>
               <term><emphasis role="bold">optional</emphasis></term>
 
               <listitem>
+                <para>This option indicates that the firewall should be able
+                to start, even if the interface is not usable for handling
+                traffic. It allows use of the <command>enable</command> and
+                <command>disable</command> commands on the interface.</para>
+
                 <para>When <option>optional</option> is specified for an
                 interface, Shorewall will be silent when:</para>
 
@@ -674,6 +677,16 @@ loc     eth2            -</programlisting>
                     <para>The first address of the interface cannot be
                     obtained.</para>
                   </listitem>
+
+                  <listitem>
+                    <para>The gateway of the interface can not be obtained
+                    (provider interface).</para>
+                  </listitem>
+
+                  <listitem>
+                    <para>The interface has been disabled using the
+                    <command>disable</command> command.</para>
+                  </listitem>
                 </itemizedlist>
 
                 <para>May not be specified with <emphasis
@@ -826,9 +839,9 @@ loc     eth2            -</programlisting>
 
                 <important>
                   <para>If ROUTE_FILTER=Yes in <ulink
-                  url="shorewall.conf.html">shorewall.conf</ulink>(5),
-                  or if your distribution sets net.ipv4.conf.all.rp_filter=1
-                  in <filename>/etc/sysctl.conf</filename>, then setting
+                  url="shorewall.conf.html">shorewall.conf</ulink>(5), or if
+                  your distribution sets net.ipv4.conf.all.rp_filter=1 in
+                  <filename>/etc/sysctl.conf</filename>, then setting
                   <emphasis role="bold">routefilter</emphasis>=0 in an
                   <replaceable>interface</replaceable> entry will not disable
                   route filtering on that
@@ -848,8 +861,8 @@ loc     eth2            -</programlisting>
                   <itemizedlist>
                     <listitem>
                       <para>If USE_DEFAULT_RT=Yes in <ulink
-                      url="shorewall.conf.html">shorewall.conf</ulink>(5)
-                      and the interface is listed in <ulink
+                      url="shorewall.conf.html">shorewall.conf</ulink>(5) and
+                      the interface is listed in <ulink
                       url="shorewall-providers.html">shorewall-providers</ulink>(5).</para>
                     </listitem>
 

docs/Build.xml

@@ -40,7 +40,11 @@
 
   <note>
     <para>This information is provided primarily for Shorewall developers.
-    Users are expected to install from pre-built tarballs or packages.</para>
+	    Users are expected to install from pre-built tarballs or packages.
+	    In addition to the below, it is also suggested to read the
+	    <ulink url="https://gitlab.com/shorewall/tools/raw/master/files/shorewall-release-process.txt">README file</ulink>
+	    located in the root directory of the tools repository.
+    </para>
   </note>
 
   <section>
@@ -98,6 +102,21 @@
       version.</para>
     </section>
 
+    <section>
+      <title>release (Clone of Release)</title>
+
+      <para>Added in Shorewall 4.4.22, this directory contains the files that
+      contain release-dependent information (change.txt, releasenotes.txt,
+      .spec files, etc). This is actually a symbolic link to ../release which
+      has its own Git repository.</para>
+    </section>
+
+    <section>
+      <title>testing (Clone of Testing)</title>
+
+      <para> This directory contains the regression library files.</para>
+    </section>
+
     <section>
       <title>tools (Clone of Tools)</title>
 
@@ -117,7 +136,8 @@
           <term>tools/files</term>
 
           <listitem>
-            <para>Files that are used during the release process.</para>
+		  <para>Files that are used during the release process.
+		  The license and readme files are also kept there.</para>
           </listitem>
         </varlistentry>
 
@@ -145,15 +165,6 @@
       <para>The files from the web site that are maintained in HTML format.
       are kept in this directory.</para>
     </section>
-
-    <section>
-      <title>release (Clone of Release)</title>
-
-      <para>Added in Shorewall 4.4.22, this directory contains the files that
-      contain release-dependent information (change.txt, releasenotes.txt,
-      .spec files, etc). This is actually a symbolic link to ../release which
-      has its own Git repository.</para>
-    </section>
   </section>
 
   <section>
@@ -180,10 +191,11 @@
     </section>
 
     <section>
-      <title>build45, build46 and build50</title>
+      <title>build45, build46, and build</title>
 
       <para>These are the scripts that respectively build Shorewall 4.5,
-      Shorewall 4.6 and Shorewall 5.[012] packages from Git.</para>
+      Shorewall 4.6 and Shorewall 5.[012] packages from Git.
+      Build is actually a symlink to the current build script.</para>
 
       <para>The scripts copy content from Git using the <command>git
       archive</command> command. They then use that content to build the
@@ -297,7 +309,7 @@
       <para>The general form of the build command is:</para>
 
       <blockquote>
-        <para><command>build</command>xx [ -<replaceable>options</replaceable>
+	      <para><command>build</command>[<replaceable>xx</replaceable>] [ -<replaceable>options</replaceable>
         ] <replaceable>release</replaceable> [ <replaceable>prior
         release</replaceable> ]</para>
       </blockquote>