14 Commits f21f257c1b ... 6234b05a50

Author SHA1 Message Date
  Alex Schroeder 6234b05a50 [usemod] Switch ISBN lookups to Wikipedia 11 months ago
  Alex Schroeder 567ea8e0a8 Fix rollback-hang tests 11 months ago
  Alex Schroeder 0974b7bbd8 wordcount: fix test 1 year ago
  Alex Schroeder f73d420957 markdown-rule: be more lenient 1 year ago
  Alex Schroeder 17ef2aaf88 CapnDan tells me this line is missing at the end 1 year ago
  Alex Schroeder b70c8e8def Add rolblack marker stripping back in 1 year ago
  Alex Schroeder f8752e69bc Update years 1 year ago
  Alex Schroeder 9d48f875a2 Fix rollback code 1 year ago
  Alex Schroeder 39e9cea7b0 Add Matched sub 1 year ago
  Alex Schroeder e7b718f610 [network-blocker] Ignore missing file 1 year ago
  Alex Schroeder 261aeccb3f [network-blocker] New module 1 year ago
  Alex Schroeder a09c846700 Fix a rollback issue 1 year ago
  Alex Schroeder 8dbede3813 Tarballs doesn't link to a latest.tar.gz 1 year ago
  Alex Schroeder 89d9f27b2a [rename-pages] Close form 1 year ago

+ 2 - 4
modules/calendar.pl

@@ -1,4 +1,4 @@
-# Copyright (C) 2004, 2005, 2006  Alex Schroeder <alex@emacswiki.org>
+# Copyright (C) 2004–2023  Alex Schroeder <alex@gnu.org>
 # Copyright (C) 2006  Ingo Belka
 #
 # This program is free software; you can redistribute it and/or modify
@@ -112,9 +112,7 @@ sub DoCollect {
   my $search = GetParam('search', '');
   ReportError(T('The match parameter is missing.')) unless $match or $search;
   print GetHeader('', Ts('Page Collection for %s', $match||$search), '');
-  my @pages = (grep(/$match/, $search
-		    ? SearchTitleAndBody($search)
-		    : AllPagesList()));
+  my @pages = Matched($match, $search ? SearchTitleAndBody($search) : AllPagesList());
   if (!$CollectingJournal) {
     $CollectingJournal = 1;
     # Now save information required for saving the cache of the current page.

+ 2 - 5
modules/index.pl

@@ -1,4 +1,4 @@
-# Copyright (C) 2004, 2007  Alex Schroeder <alex@emacswiki.org>
+# Copyright (C) 2004–2023  Alex Schroeder <alex@gnu.org>
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -47,8 +47,5 @@ sub PrintableIndexPages {
   push(@pages, AllPagesList()) if GetParam('pages', 1);
   push(@pages, keys %PermanentAnchors) if GetParam('permanentanchors', 1);
   push(@pages, keys %NearSource) if GetParam('near', 0);
-  my $match = GetParam('match', '');
-  @pages = grep /$match/i, @pages if $match;
-  @pages = sort @pages;
-  return @pages;
+  return sort Matched(GetParam('match'), @pages);
 }

+ 2 - 2
modules/journal-rss.pl

@@ -1,4 +1,4 @@
-# Copyright (C) 2004–2021  Alex Schroeder <alex@gnu.org>
+# Copyright (C) 2004–2023  Alex Schroeder <alex@gnu.org>
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -86,7 +86,7 @@ sub JournalRssGetRcLines {
   my $reverse = GetParam('reverse', 0);
   my $monthly = GetParam('monthly', 0);
   my $offset = GetParam('offset', 0);
-  my @pages = sort JournalSort (grep(/$match/, $search ? SearchTitleAndBody($search) : AllPagesList()));
+  my @pages = sort JournalSort (Matched($match, $search ? SearchTitleAndBody($search) : AllPagesList()));
   if ($monthly and not $match) {
     my ($sec, $min, $hour, $mday, $mon, $year, $wday, $yday) = gmtime();
     $match = '^' . sprintf("%04d-%02d", $year+1900, $mon+1) . '-\d\d';

+ 1 - 1
modules/mail.pl

@@ -1,4 +1,4 @@
-# Copyright (C) 2009–2020  Alex Schroeder <alex@gnu.org>
+# Copyright (C) 2009–2022  Alex Schroeder <alex@gnu.org>
 # Copyright (C) 2015 Aleks-Daniel Jakimenko <alex.jakimenko@gmail.com>
 #
 # This program is free software; you can redistribute it and/or modify it under

+ 3 - 4
modules/markdown-rule.pl

@@ -1,5 +1,5 @@
 #! /usr/bin/perl
-# Copyright (C) 2014–2019  Alex Schroeder <alex@gnu.org>
+# Copyright (C) 2014–2022  Alex Schroeder <alex@gnu.org>
 
 # This program is free software: you can redistribute it and/or modify it under
 # the terms of the GNU General Public License as published by the Free Software
@@ -194,12 +194,11 @@ sub MarkdownRule {
     return OpenHtmlEnvironment('pre',1) . $str; # always level 1
   }
   # link: [an example](http://example.com/ "Title")
-  elsif (m/\G\[((?:[^]\n]+\n?)+)\]\($FullUrlPattern(\s+"(.+?)")?\)/cg) {
+  elsif (m/\G\[((?:[^]\n]+\n?)+)\]\((\S+)(\s+"(.+?)")?\)/cg) {
     my ($text, $url, $title) = ($1, $2, $4);
-    $url =~ /^($UrlProtocols)/;
     my %params;
     $params{-href} = $url;
-    $params{-class} = "url $1";
+    $params{-class} = "url";
     $params{-title} = $title if $title;
     return $q->a(\%params, $text);
   }

+ 1 - 1
modules/namespaces.pl

@@ -1,4 +1,4 @@
-# Copyright (C) 2004, 2005, 2006, 2007, 2008, 2009, 2012  Alex Schroeder <alex@gnu.org>
+# Copyright (C) 2004–2022  Alex Schroeder <alex@gnu.org>
 #
 # This program is free software; you can redistribute it and/or modify it under
 # the terms of the GNU General Public License as published by the Free Software

+ 198 - 0
modules/network-blocker.pl

@@ -0,0 +1,198 @@
+# -*- mode: perl -*-
+# Copyright (C) 2023  Alex Schroeder <alex@gnu.org>
+
+# This program is free software: you can redistribute it and/or modify it under
+# the terms of the GNU Affero General Public License as published by the Free
+# Software Foundation, either version 3 of the License, or (at your option) any
+# later version.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+# details.
+#
+# You should have received a copy of the GNU Affero General Public License along
+# with this program. If not, see <https://www.gnu.org/licenses/>.
+
+=encoding utf8
+
+=head1 NAME
+
+Oddmuse Network Blocker
+
+=head1 DESCRIPTION
+
+This module hooks into regular Oddmuse Surge Protection. It adds the following
+features:
+
+Repeated offenders are blocked for increasingly longer times.
+
+For every offender, we record the CIDR their IP number belongs to. Everytime an
+IP number is blocked, all the CIDRs of the other blocked IPs are checked: if
+there are three or more blocked IP numbers sharing the same CIDRs, the CIDR
+itself is blocked.
+
+CIDR blocking works the same way: Repeated offenders are blocked for
+increasingly longer times.
+
+=head2 Behind a reverse proxy
+
+Make sure your config file copies the IP number to the correct environment
+variable:
+
+    $ENV{REMOTE_ADDR} = $ENV{HTTP_X_FORWARDED_FOR};
+
+=head1 SEE ALSO
+
+<Oddmuse Surge Protection|https://oddmuse.org/wiki/Surge_Protection>
+
+=cut
+
+use strict;
+use v5.10;
+use Net::IP qw(:PROC);
+use Net::DNS qw(rr);
+
+AddModuleDescription('network-blocker.pl', 'Network Blocker Extension');
+
+our ($Now, $DataDir, $SurgeProtectionViews, $SurgeProtectionTime);
+
+{
+  no warnings 'redefine';
+  *OldNetworkBlockerDelayRequired = \&DelayRequired;
+  *DelayRequired = \&NewNetworkBlockerDelayRequired;
+}
+
+# Block for at least this many seconds.
+my $NetworkBlockerMinimumPeriod = 30;
+
+# Every violation doubles the current period until this maximum is reached (four weeks).
+my $NetworkBlockerMaximumPeriod = 60 * 60 * 24 * 7 * 4;
+
+# All the blocked networks. Maps CIDR to an array [expiry timestamp, expiry
+# period].
+my %NetworkBlockerList;
+
+# Candidates are remembered for this many seconds.
+my $NetworkBlockerCachePeriod = 600;
+
+# All the candidate networks for a block. Maps IP to an array [ts, cidr, ...].
+# Candidates are removed after $NetworkBlockerCachePeriod.
+my %NetworkBlockerCandidates;
+
+sub NetworkBlockerRead {
+  my ($status, $data) = ReadFile("$DataDir/network-blocks");
+  return unless $status;
+  my @lines = split(/\n/, $data);
+  while ($_ = shift(@lines)) {
+    my @items = split(/,/);
+    $NetworkBlockerList{shift(@items)} = \@items;
+  }
+  # an empty line separates the two sections
+  while ($_ = shift(@lines)) {
+    my @items = split(/,/);
+    $NetworkBlockerCandidates{shift(@items)} = \@items;
+  }
+  return 1;
+}
+
+sub NetworkBlockerWrite {
+  RequestLockDir('network-blocks') or return '';
+  WriteStringToFile(
+    "$DataDir/network-blocks",
+    join("\n\n",
+         join("\n", map {
+           join(",", $_, @{$NetworkBlockerList{$_}})
+              } keys %NetworkBlockerList),
+         join("\n", map {
+           join(",", $_, @{$NetworkBlockerCandidates{$_}})
+              } keys %NetworkBlockerCandidates)));
+  ReleaseLockDir('network-blocks');
+}
+
+sub NewNetworkBlockerDelayRequired {
+  my $ip = shift;
+  # If $ip is a name and not an IP number, parsing fails. In this case, run the
+  # regular code.
+  my $ob = new Net::IP($ip);
+  return OldNetworkBlockerDelayRequired($ip) unless $ob;
+  # Read the file. If the file does not exist, no problem.
+  NetworkBlockerRead();
+  # See if the current IP number is one of the blocked CIDR ranges.
+  for my $cidr (keys %NetworkBlockerList) {
+    # Perhaps this CIDR block can be expired.
+    if ($NetworkBlockerList{$cidr}->[0] < $Now) {
+      delete $NetworkBlockerList{$cidr};
+      next;
+    }
+    # Forget the CIDR if it cannot be turned into a range.
+    my $range = new Net::IP($cidr);
+    if (not $range) {
+      warn "CIDR $cidr is blocked but has no range: " . Net::IP::Error();
+      delete $NetworkBlockerList{$cidr};
+      next;
+    }
+    # If the CIDR overlaps with the remote IP number, it's a block.
+    warn "Checking whether $ip is in $cidr\n";
+    my $overlap = $range->overlaps($ob);
+    # $IP_PARTIAL_OVERLAP (ranges overlap) $IP_NO_OVERLAP (no overlap)
+    # $IP_A_IN_B_OVERLAP (range2 contains range1) $IP_B_IN_A_OVERLAP (range1
+    # contains range2) $IP_IDENTICAL (ranges are identical) undef (problem)
+    if (defined $overlap and $overlap != $IP_NO_OVERLAP) {
+      # Double the block period unless it has reached $NetworkBlockerMaximumPeriod.
+      if ($NetworkBlockerList{$cidr}->[1] < $NetworkBlockerMaximumPeriod / 2) {
+        $NetworkBlockerList{$cidr}->[1] *= 2;
+      } else {
+        $NetworkBlockerList{$cidr}->[1] = $NetworkBlockerMaximumPeriod;
+      }
+      $NetworkBlockerList{$cidr}->[0] = $Now + $NetworkBlockerList{$cidr}->[1];
+      # And we're done!
+      NetworkBlockerWrite();
+      ReportError(Ts('Too many connections by %s', $cidr)
+		  . ': ' . Tss('Please do not fetch more than %1 pages in %2 seconds.',
+			       $SurgeProtectionViews, $SurgeProtectionTime),
+		  '503 SERVICE UNAVAILABLE');
+    }
+  }
+  # If the CIDR isn't blocked, let's see if Surge Protection wants to block it.
+  my $result = OldNetworkBlockerDelayRequired($ip);
+  warn "$ip was blocked\n" if $result;
+  # If the IP is to be blocked, determine its CIDRs and put them on a list. Sadly,
+  # routeviews does not support IPv6 at the moment!
+  if ($result and not ip_is_ipv6($ip) and not $NetworkBlockerCandidates{$ip}) {
+    my $reverse = $ob->reverse_ip();
+    $reverse =~ s/in-addr\.arpa\.$/asn.routeviews.org/;
+    my @candidates;
+    for my $rr (rr($reverse, "TXT")) {
+      next unless $rr->type eq "TXT";
+      my @data = $rr->txtdata;
+      push(@candidates, join("/", @data[1..2]));
+    }
+    warn "$ip is in @candidates\n";
+    $NetworkBlockerCandidates{$ip} = [$Now, @candidates];
+    # Expire any of the other candidates
+    for my $other_ip (keys %NetworkBlockerCandidates) {
+      if ($NetworkBlockerCandidates{$other_ip}->[0] < $Now - $NetworkBlockerCachePeriod) {
+        delete $NetworkBlockerCandidates{$other_ip};
+      }
+    }
+    # Determine if any of the CIDRs is to be blocked.
+    my $save;
+    for my $cidr (@candidates) {
+      # Count how often the candidate CIDRs show up for other IP numbers.
+      my $count = 0;
+      for my $other_ip (keys %NetworkBlockerCandidates) {
+        my @data = $NetworkBlockerCandidates{$other_ip};
+        for my $other_cidr (@data[1 .. $#data]) {
+          $count++ if $cidr eq $other_cidr;
+        }
+      }
+      if ($count >= 3) {
+        $NetworkBlockerList{$cidr} = [$Now + $NetworkBlockerMinimumPeriod, $NetworkBlockerMinimumPeriod];
+        $save = 1;
+      }
+    }
+    NetworkBlockerWrite() if $save;
+  }
+  return $result;
+}

+ 3 - 2
modules/rename-pages.pl

@@ -1,4 +1,4 @@
-# Copyright (C) 2019  Alex Schroeder <alex@gnu.org>
+# Copyright (C) 2019–2023  Alex Schroeder <alex@gnu.org>
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -65,6 +65,7 @@ sub RenamePageMenu {
 	 . GetHiddenValue('id', $id)
 	 . $q->textfield(-name=>'to', -size=>20)
 	 . ' '
-	 . $q->submit('Do it'));
+	 . $q->submit('Do it')
+         . $q->end_form());
   }
 }

+ 16 - 30
modules/search-list.pl

@@ -1,4 +1,4 @@
-# Copyright (C) 2006, 2007, 2008  Alex Schroeder <alex@emacswiki.org>
+# Copyright (C) 2006–2023  Alex Schroeder <alex@gnu.org>
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -36,18 +36,12 @@ sub SearchListRule {
       $term = GetId();
     }
     local ($OpenPageName, %Page);
-    my %hash = ();
+    my @found;
     if ($variation eq 'list') {
-      foreach my $id (SearchTitleAndBody($term)) {
-        $hash{$id} = 1 unless $id eq $original; # skip the page with the query
-      }
+      @found = grep { $_ ne $original } SearchTitleAndBody($term);
+    } elsif ($variation eq 'titlelist') {
+      @found = grep { $_ ne $original } Matched($term, AllPagesList());
     }
-    if ($variation eq 'titlelist') {
-      foreach my $id (grep(/$term/, AllPagesList())) {
-        $hash{$id} = 1 unless $id eq $original; # skip the page with the query
-      }
-    }
-    my @found = keys %hash;
     if (defined &PageSort) {
       @found = sort PageSort @found;
     } else {
@@ -63,32 +57,24 @@ sub SearchListRule {
   return;
 }
 
-
 # Add a new action list
 
 $Action{list} = \&DoList;
 
 sub DoList {
-my $id = shift;
-my $match = GetParam('match', '');
-my $search = GetParam('search', '');
+  my $id = shift;
+  my $match = GetParam('match', '');
+  my $search = GetParam('search', '');
   ReportError(T('The search parameter is missing.')) unless $match or $search;
   print GetHeader('', Ts('Page list for %s', $match||$search), '');
   local (%Page, $OpenPageName);
-    my %hash = ();
-    foreach my $id (grep(/$match/, $search
-                    ? SearchTitleAndBody($search)
-                    : AllPagesList()))  {
-      $hash{$id} = 1;
-    }
-    my @found = keys %hash;
-    if (defined &PageSort) {
-      @found = sort PageSort @found;
-    } else {
-      @found = sort(@found);
-    }
-    @found = map { $q->li(GetPageLink($_)) } @found;
-    print $q->start_div({-class=>'search list'}),
-      $q->ul(@found), $q->end_div;
+  my @found = Matched($match, $search ? SearchTitleAndBody($search) : AllPagesList());
+  if (defined &PageSort) {
+    @found = sort PageSort @found;
+  } else {
+    @found = sort(@found);
+  }
+  @found = map { $q->li(GetPageLink($_)) } @found;
+  print $q->start_div({-class=>'search list'}), $q->ul(@found), $q->end_div;
   PrintFooter();
 }

+ 0 - 0
modules/sistersites.pl


Some files were not shown because too many files changed in this diff