Additions to release notes; only split CONFIG_PATH once; don't recompile REs for each tcrule

git-svn-id: https://shorewall.svn.sourceforge.net/svnroot/shorewall/trunk@5762 fbd18981-670d-0410-9b5c-8dc0c1a9a2bb

New/Shorewall/Chains.pm

@@ -120,6 +120,7 @@ our @VERSION = 1.00;
 #    @policy_chains is a list of references to policy chains in the filter table
 #
 #    %chain_table { <table> => { <chain1>  => { name         => <chain name>
+#                                               table        => <table name>
 #                                               is_policy    => 0|1
 #                                               is_optionsl  => 0|1
 #                                               referenced   => 0|1      
@@ -132,7 +133,8 @@ our @VERSION = 1.00;
 #                                                                 <rule2>
 #                                                                 ...
 #                                                               ]
-#                                             }
+#                                             } ,
+#                                <chain2> => ...
 #                              }
 #                 }
 #
@@ -692,6 +694,7 @@ sub mac_match( $ ) {
 #
 sub numeric_value ( $ ) {
     my $mark = $_[0];
+    fatal_error "Invalid Numeric Value" unless "\L$mark" =~ /^(0x[a-f0-9]+|0[0-7]*|[1-9]\d*)$/;
     $mark =~ /^0x/ ? hex $mark : $mark =~ /^0/ ? oct $mark : $mark;
 }
 
@@ -703,7 +706,7 @@ sub verify_mark( $ ) {
     my $limit = $config{HIGH_ROUTE_MARKS} ? 0xFFFF : 0xFF;
 
     fatal_error "Invalid Mark or Mask value: $mark" 
-	unless "\L$mark" =~ /^(0x[a-f0-9]+|0[0-7]*|[0-9]*)$/ && numeric_value( $mark ) <= $limit;
+	unless numeric_value( $mark ) <= $limit;
 }
 
 sub verify_small_mark( $ ) {
@@ -837,7 +840,6 @@ sub match_source_net( $ ) {
  
     if ( $net =~ /^(!?).*\..*\..*\..*-.*\..*\..*\..*/ ) {
 	$net =~ s/!// if my $invert = $1 ? '! ' : '';
-
 	iprange_match . "${invert}--src-range $net ";
     } elsif ( $net =~ /^(!?)~(.*)$/ ) {
 	( $net = $2 ) =~ s/-/:/g;
@@ -861,7 +863,6 @@ sub match_dest_net( $ ) {
 
     if ( $net =~ /^(!?).*\..*\..*\..*-.*\..*\..*\..*/ ) {
 	$net =~ s/!// if my $invert = $1 ? '! ' : '';
-
 	iprange_match . "${invert}--dst-range $net ";
     } elsif ( $net =~ /^(!?)\+/ ) {
 	require_capability( 'IPSET_MATCH' , 'ipset names in Shorewall configuration files' );

New/Shorewall/Config.pm

@@ -215,6 +215,7 @@ my %capdesc = ( NAT_ENABLED     => 'NAT',
 		ADDRTYPE        => 'Address Type Match',
 		);
 
+my @config_path;
 #
 # Stash away file references here when we encounter INCLUDE
 #
@@ -255,8 +256,7 @@ sub find_file($)
 
     my $directory;
 
-    for $directory ( split ':', $ENV{CONFIG_PATH} ) {
+    for $directory ( @config_path ) {
-	$directory = "$directory/" unless substr( $directory, -1, 1 ) eq '/';
 	my $file = "$directory$filename";
 	return $file if -f $file;
     }
@@ -294,17 +294,17 @@ sub split_line( $$ ) {
 }
 
 #
-# Some files can have shell variables embedded. This function expands them from %ENV.
+# Config files can have shell variables embedded. This function expands them from %ENV.
 #
 sub expand_shell_variables( $ ) {
     my $line = $_[0]; 
-    $line = $1 . ( $ENV{$2} || '' ) . $3 while $line =~ /^(.*?)\${([a-zA-Z]\w*)}(.*)$/;
+    $line = join( '', $1 , ( $ENV{$2} || '' ) , $3 ) while $line =~ /^(.*?)\${([a-zA-Z]\w*)}(.*)$/;
-    $line = $1 . ( $ENV{$2} || '' ) . $3 while $line =~ /^(.*?)\$([a-zA-Z]\w*)(.*)$/;
+    $line = join( '', $1 , ( $ENV{$2} || '' ) , $3 ) while $line =~ /^(.*?)\$([a-zA-Z]\w*)(.*)$/;
     $line;
 }
 
 #
-# Open a file, setting $currentfile. Returns the absolute pathname if the file
+# Open a file, setting $currentfile. Returns the file's absolute pathname if the file
 # exists, is non-empty  and was successfully opened. Terminates with a fatal error
 # if the file exists, is non-empty, but the open fails.
 #
@@ -466,6 +466,13 @@ sub require_capability( $$ ) {
 # - establish global hashes %config , %env and %capabilities
 #
 sub get_configuration() {
+
+    @config_path = split /:/, $ENV{CONFIG_PATH};
+
+    for ( @config_path ) {
+	$_ .= '/' unless m|//$|;
+    }
+
     my $file = find_file 'shorewall.conf';
 
     if ( -f $file ) {

New/Shorewall/Rules.pm

@@ -324,12 +324,11 @@ sub setup_blacklist() {
 
 sub process_criticalhosts() {
 
-    my $fn = find_file 'routestopped';
+    my  @critical = ();
-    my  @critical;
 
-    @critical = ();
+    my $fn = open_file $fn;
 
-    open_file $fn;
+    my $first_entry = 1;
 
     while ( read_a_line ) {
 
@@ -337,6 +336,11 @@ sub process_criticalhosts() {
 
 	my ($interface, $hosts, $options ) = split_line 3, 'routestopped file';
 
+	if ( $first_entry ) {
+	    progress_message2 "$doing $fn for critical hosts...";
+	    $first_entry = 0;
+	}
+
 	$hosts = ALLIPv4 unless $hosts && $hosts ne '-';
 
 	my @hosts;

New/Shorewall/Tc.pm

@@ -92,26 +92,26 @@ use constant { NOMARK    => 0 ,
 	       HIGHMARK  => 2 
 	       };
 
-my @tccmd = ( { pattern   => 'SAVE' ,
+my @tccmd = ( { match     => sub ( $ ) { $_[0] eq 'SAVE' } ,
 		target    => 'CONNMARK --save-mark --mask' ,
 		mark      => SMALLMARK ,
 		mask      => '0xFF'
 		} ,
-	      { pattern   => 'RESTORE' ,
+	      { match     => sub ( $ ) { $_[0] eq 'RESTORE' },
 		target    => 'CONNMARK --restore-mark --mask' ,
 		mark      => SMALLMARK ,
 		mask      => '0xFF'
 		} ,
-	      { pattern   => 'CONTINUE',
+	      { match     => sub ( $ ) { $_[0] eq 'CONTINUE' },
 		target    => 'RETURN' ,
 		mark      => NOMARK ,
 		mask      => '' 
 		} ,
-	      { pattern   => '\|.*' ,
+	      { match     => sub ( $ ) { $_[0] =~ '\|.*'} ,
 		target    => 'MARK --or-mark' ,
 		mark      => HIGHMARK ,
 		mask      => '' } ,
-	      { pattern   => '&.*' ,
+	      { match     => sub ( $ ) { $_[0] =~ '&.*' },
 		target    => 'MARK --and-mark ' ,
 		mark      => HIGHMARK ,
 		mask      => '' 
@@ -171,7 +171,7 @@ sub process_tc_rule( $$$$$$$$$$ ) {
 	    {
 	  PATTERN:
 		for my $tccmd ( @tccmd ) {
-		    if ( $cmd =~ /^($tccmd->{pattern})$/ ) {
+		    if ( $tccmd->{match}($cmd) ) {
 			fatal_error "$mark not valid with :C[FP]" if $connmark;
 
 			$target      = "$tccmd->{target} ";

New/releasenotes.txt

@@ -30,10 +30,10 @@ get sticky.
 
 The good news:
 
-a) The compiler is small.
+a) The compiler has a small disk footprint.
 b) The compiler is very fast.
 c) The compiler generates a firewall script that uses iptables-restore;
-so the script is very fast.
+   so the script is very fast.
 d) Use of the perl compiler is optional! The old slow clunky
    Bourne-shell compiler is still available.
 
@@ -51,28 +51,33 @@ a) The Perl-based compiler requires the following capabilities in your
 
    These capabilities are in current distributions.
 
-b) The Bourne-shell compiler goes to great pain (in some cases) to
+b) Now that Netfilter has features to deal reasonably with port lists,
+   I see no reason to duplicate those features in Shorewall. The
+   Bourne-shell compiler goes to great pain (in some cases) to
    break very long port lists ( > 15 where port ranges in lists count
    as two ports) into individual rules. In the new compiler, I'm
    avoiding the ugliness required to do that. The new compiler just
-   gives you an error if your list is too long. It will also give you
+   generates an error if your list is too long. It will also produce
    an error if you insert a port range into a port list and you don't
-   have extended multiport support. Now that Netfilter has features to
+   have extended multiport support. 
-   deal reasonably with port lists, I see no reason to duplicate those
-   features in Shorewall.
 
 c) BRIDGING=Yes is not supported. The kernel code necessary to
    support this option was removed in Linux kernel 2.6.20. 
 
-d) The BROADCAST column in the interfaces file is essentailly unused;
+d) The BROADCAST column in the interfaces file is essentially unused;
    if you enter anything in this column but '-' or 'detect', you will
    receive a warning. This will be relaxed if and when the addrtype
    match requirement is relaxed.
 
 e) Because the compiler is now written in Perl, your compile-time
-   extension scripts from earlier versions will no longer work.
+   extension scripts from earlier versions will no longer work. For
+   now, if you want to use extension scripts, you will need to read the
+   Perl code to see how the compiler operates internally. I will
+   produce documentation before the first official release.
+   Compile-time extension scripts are executed using the Perl
+   'do FILE' mechanism.
 
-f) The 'refresh' command is now synonamous with 'restart'.
+f) The 'refresh' command is now synonymous with 'restart'.
 
 g) Some run-time extension scripts are no longer supported because they
    make no sense (iptables-restore instantiates the new configuration
@@ -97,23 +102,39 @@ h) The /etc/shorewall/tos file now has zone-independent SOURCE and DEST
        [all:]<address>[,...]
        [all:]<interface>[:<address>[,...]]
 
-   This is a perminent change. The old zone-based rules have never
+   This is a permanent change. The old zone-based rules have never
    worked right and this is a good time to replace them. I've tried to
    make the new syntax cover the most common cases without requiring
    change to existing files. In particular, it will handle the tos file
    released with Shorewall 1.4 and earlier.
 
 i) Currently, support for ipsets is untested. That will change with
-   future releases but one thing is certain -- Shorewall is now out of the
+   future pre-releases but one thing is certain -- Shorewall is now out
-   ipset load/reload business. With scripts generated by the Perl-based
+   of the ipset load/reload business. With scripts generated by the
-   Compiler, the Netfilter ruleset is never cleared. That means that
+   Perl-based Compiler, the Netfilter ruleset is never cleared. That
-   there is no opportunity for Shorewall to load/reload your ipsets
+   means that there is no opportunity for Shorewall to load/reload your
-   since that cannot be done while there are any current rules using
+   ipsets since that cannot be done while there are any current rules
-   your ipsets.
+   using ipsets.
 
    So:
+	i)   Your ipsets must be loaded before Shorewall starts. You
+	     are free to try to do that with the following code in
+	     /etc/shorewall/start:
 
-	i)   Your ipsets must be loaded before Shorewall starts.
+	     if [ "$COMMAND" = start ]; then
+		ipset -U :all: :all:
+		ipset -F
+		ipset -X
+		ipset -R < /my/ipset/contents
+	     fi
+
+	     The file '/my/ipset/contents' (not its real name of
+	     course) will normally be produced using the ipset -S
+	     command.
+
+	     The above will work most of the time but will fail in a 
+	     'shorewall stop' - 'shorewall start' sequence if you 
+	     use ipsets in your routestopped file (see below).
 	
 	ii)  Your ipsets may not be reloaded until Shorewall is stopped or
 	     cleared.
@@ -125,6 +146,31 @@ i) Currently, support for ipsets is untested. That will change with
    ignore /etc/shorewall/ipsets and will issue a warning if you set
    SAVE_IPSETS=Yes in shorewall.conf.
 
+j) Because the configuration files (with the exception of
+   /etc/shorewall/params) are now processed by the Perl-based compiler
+   rather than by the shell, only the basic forms of Shell expansion
+   ($variable and ${variable}) are supported. The more exotic forms
+   such as ${variable:=default} are not supported. Both variables
+   defined in /etc/shorewall/params and environmental variables
+   (exported by the shell) can be used.
+
+h) Line continuation (lines ending in '\' are concatinated with the next
+   line) continues to be supported. Unlike the shell-based compiler,
+   however, the Perl-based compiler does not continue lines that end
+   in '#' comments. This avoids the confusing behavior where
+   the last line of a comment ends with '\', causing the
+   next (non-comment) line to be ignored.
+
+   Example (/etc/shorewall/tunnels):
+
+   # VPN from Atlanta \
+   openvpn-server     net	206.124.146.177    
+   
+   With the Shell-based compiler, the openvpn-server line is ignored
+   because it is combined with the preceding line; with the Perl-based
+   compiler, it is processed normally.
+
+
 Installation
 ------------