treat asterisk as junk; treat repeating characters as junk

sciencebeam_gym/preprocess/annotation/fuzzy_match.py

@@ -30,7 +30,9 @@ def len_index_range(index_range):
 # Treat space or comma after a dot, or a dot after a letter as junk
 DEFAULT_ISJUNK = lambda s, i: (
   (i > 0 and s[i - 1] == '.' and (s[i] == ' ' or s[i] == ',')) or
-  (i > 0 and s[i - 1].isalpha() and s[i] == '.')
+  (i > 0 and s[i - 1].isalpha() and s[i] == '.') or
+  (i > 0 and s[i - 1] == s[i]) or
+  s[i] == '*'
 )
 
 DOT_IS_JUNK = lambda s, i: s[i] == '.'

sciencebeam_gym/preprocess/annotation/fuzzy_match_test.py

@@ -20,6 +20,12 @@ class TestRemoveJunk(object):
   def test_should_remove_dots_after_capitals(self):
     assert remove_junk('P.O. Box', DOT_IS_JUNK) == 'PO Box'
 
+  def test_should_remove_asterisk_after_capitals(self):
+    assert remove_junk('Mr Beam*') == 'Mr Beam'
+
+  def test_should_remove_repeating_characters(self):
+    assert remove_junk('Mr Beeeam') == 'Mr Beam'
+
 class TestInvertIndexRanges(object):
   def test_should_return_empty_for_empty_range(self):
     assert list(invert_index_ranges([], 0, 0)) == list([])