mirror of
https://github.com/sphinx-doc/sphinx.git
synced 2026-10-12 08:25:01 -05:00
Fetch Snowball stopword lists as of 2015-09-01
This commit is contained in:
Adam Turner committed
1 parent
9a41636348
commit
f265d50a14
27 files changed
+606
-30
No files matched your search
@@ -1,3 +1,6 @@
|
||||
# automatically generated by utils/generate-snowball.py
|
||||
# from https://github.com/snowballstem/snowball-website/raw/efb4ae4d65769fb1652acbe608c0c817e746c730/algorithms/danish/stop.txt
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
DANISH_STOPWORDS = frozenset({
|
||||
|
||||
@@ -1,4 +1,11 @@
|
||||
| source: https://snowballstem.org/algorithms/danish/stop.txt
|
||||
|
||||
| A Danish stop word list. Comments begin with vertical bar. Each stop
|
||||
| word is at the start of a line.
|
||||
|
||||
| This is a ranked list (commonest to rarest) of stopwords derived from
|
||||
| a large text sample.
|
||||
|
||||
|
||||
og | and
|
||||
i | in
|
||||
jeg | I
|
||||
@@ -45,7 +52,7 @@ hvor | where
|
||||
eller | or
|
||||
hvad | what
|
||||
skal | must/shall etc.
|
||||
selv | myself/yourself/herself/ourselves etc., even
|
||||
selv | myself/youself/herself/ourselves etc., even
|
||||
her | here
|
||||
alle | all/everyone/everybody etc.
|
||||
vil | will (verb)
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
# automatically generated by utils/generate-snowball.py
|
||||
# from https://github.com/snowballstem/snowball-website/raw/efb4ae4d65769fb1652acbe608c0c817e746c730/algorithms/german/stop.txt
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
GERMAN_STOPWORDS = frozenset({
|
||||
|
||||
@@ -1,4 +1,11 @@
|
||||
|source: https://snowballstem.org/algorithms/german/stop.txt
|
||||
|
||||
| A German stop word list. Comments begin with vertical bar. Each stop
|
||||
| word is at the start of a line.
|
||||
|
||||
| The number of forms in this list is reduced significantly by passing it
|
||||
| through the German stemmer.
|
||||
|
||||
|
||||
aber | but
|
||||
|
||||
alle | all
|
||||
|
||||
@@ -1,37 +1,181 @@
|
||||
# automatically generated by utils/generate-snowball.py
|
||||
# from https://github.com/snowballstem/snowball-website/raw/efb4ae4d65769fb1652acbe608c0c817e746c730/algorithms/english/stop.txt
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
ENGLISH_STOPWORDS = frozenset({
|
||||
'a',
|
||||
'about',
|
||||
'above',
|
||||
'after',
|
||||
'again',
|
||||
'against',
|
||||
'all',
|
||||
'am',
|
||||
'an',
|
||||
'and',
|
||||
'any',
|
||||
'are',
|
||||
"aren't",
|
||||
'as',
|
||||
'at',
|
||||
'be',
|
||||
'because',
|
||||
'been',
|
||||
'before',
|
||||
'being',
|
||||
'below',
|
||||
'between',
|
||||
'both',
|
||||
'but',
|
||||
'by',
|
||||
"can't",
|
||||
'cannot',
|
||||
'could',
|
||||
"couldn't",
|
||||
'did',
|
||||
"didn't",
|
||||
'do',
|
||||
'does',
|
||||
"doesn't",
|
||||
'doing',
|
||||
"don't",
|
||||
'down',
|
||||
'during',
|
||||
'each',
|
||||
'few',
|
||||
'for',
|
||||
'from',
|
||||
'further',
|
||||
'had',
|
||||
"hadn't",
|
||||
'has',
|
||||
"hasn't",
|
||||
'have',
|
||||
"haven't",
|
||||
'having',
|
||||
'he',
|
||||
"he'd",
|
||||
"he'll",
|
||||
"he's",
|
||||
'her',
|
||||
'here',
|
||||
"here's",
|
||||
'hers',
|
||||
'herself',
|
||||
'him',
|
||||
'himself',
|
||||
'his',
|
||||
'how',
|
||||
"how's",
|
||||
'i',
|
||||
"i'd",
|
||||
"i'll",
|
||||
"i'm",
|
||||
"i've",
|
||||
'if',
|
||||
'in',
|
||||
'into',
|
||||
'is',
|
||||
"isn't",
|
||||
'it',
|
||||
'near',
|
||||
"it's",
|
||||
'its',
|
||||
'itself',
|
||||
"let's",
|
||||
'me',
|
||||
'more',
|
||||
'most',
|
||||
"mustn't",
|
||||
'my',
|
||||
'myself',
|
||||
'no',
|
||||
'nor',
|
||||
'not',
|
||||
'of',
|
||||
'off',
|
||||
'on',
|
||||
'once',
|
||||
'only',
|
||||
'or',
|
||||
'other',
|
||||
'ought',
|
||||
'our',
|
||||
'ours',
|
||||
'ourselves',
|
||||
'out',
|
||||
'over',
|
||||
'own',
|
||||
'same',
|
||||
"shan't",
|
||||
'she',
|
||||
"she'd",
|
||||
"she'll",
|
||||
"she's",
|
||||
'should',
|
||||
"shouldn't",
|
||||
'so',
|
||||
'some',
|
||||
'such',
|
||||
'than',
|
||||
'that',
|
||||
"that's",
|
||||
'the',
|
||||
'their',
|
||||
'theirs',
|
||||
'them',
|
||||
'themselves',
|
||||
'then',
|
||||
'there',
|
||||
"there's",
|
||||
'these',
|
||||
'they',
|
||||
"they'd",
|
||||
"they'll",
|
||||
"they're",
|
||||
"they've",
|
||||
'this',
|
||||
'those',
|
||||
'through',
|
||||
'to',
|
||||
'too',
|
||||
'under',
|
||||
'until',
|
||||
'up',
|
||||
'very',
|
||||
'was',
|
||||
'will',
|
||||
"wasn't",
|
||||
'we',
|
||||
"we'd",
|
||||
"we'll",
|
||||
"we're",
|
||||
"we've",
|
||||
'were',
|
||||
"weren't",
|
||||
'what',
|
||||
"what's",
|
||||
'when',
|
||||
"when's",
|
||||
'where',
|
||||
"where's",
|
||||
'which',
|
||||
'while',
|
||||
'who',
|
||||
"who's",
|
||||
'whom',
|
||||
'why',
|
||||
"why's",
|
||||
'with',
|
||||
"won't",
|
||||
'would',
|
||||
"wouldn't",
|
||||
'you',
|
||||
"you'd",
|
||||
"you'll",
|
||||
"you're",
|
||||
"you've",
|
||||
'your',
|
||||
'yours',
|
||||
'yourself',
|
||||
'yourselves',
|
||||
})
|
||||
@@ -0,0 +1,311 @@
|
||||
|
||||
| An English stop word list. Comments begin with vertical bar. Each stop
|
||||
| word is at the start of a line.
|
||||
|
||||
| Many of the forms below are quite rare (e.g. "yourselves") but included for
|
||||
| completeness.
|
||||
|
||||
| PRONOUNS FORMS
|
||||
| 1st person sing
|
||||
|
||||
i | subject, always in upper case of course
|
||||
|
||||
me | object
|
||||
my | possessive adjective
|
||||
| the possessive pronoun `mine' is best suppressed, because of the
|
||||
| sense of coal-mine etc.
|
||||
myself | reflexive
|
||||
| 1st person plural
|
||||
we | subject
|
||||
|
||||
| us | object
|
||||
| care is required here because US = United States. It is usually
|
||||
| safe to remove it if it is in lower case.
|
||||
our | possessive adjective
|
||||
ours | possessive pronoun
|
||||
ourselves | reflexive
|
||||
| second person (archaic `thou' forms not included)
|
||||
you | subject and object
|
||||
your | possessive adjective
|
||||
yours | possessive pronoun
|
||||
yourself | reflexive (singular)
|
||||
yourselves | reflexive (plural)
|
||||
| third person singular
|
||||
he | subject
|
||||
him | object
|
||||
his | possessive adjective and pronoun
|
||||
himself | reflexive
|
||||
|
||||
she | subject
|
||||
her | object and possessive adjective
|
||||
hers | possessive pronoun
|
||||
herself | reflexive
|
||||
|
||||
it | subject and object
|
||||
its | possessive adjective
|
||||
itself | reflexive
|
||||
| third person plural
|
||||
they | subject
|
||||
them | object
|
||||
their | possessive adjective
|
||||
theirs | possessive pronoun
|
||||
themselves | reflexive
|
||||
| other forms (demonstratives, interrogatives)
|
||||
what
|
||||
which
|
||||
who
|
||||
whom
|
||||
this
|
||||
that
|
||||
these
|
||||
those
|
||||
|
||||
| VERB FORMS (using F.R. Palmer's nomenclature)
|
||||
| BE
|
||||
am | 1st person, present
|
||||
is | -s form (3rd person, present)
|
||||
are | present
|
||||
was | 1st person, past
|
||||
were | past
|
||||
be | infinitive
|
||||
been | past participle
|
||||
being | -ing form
|
||||
| HAVE
|
||||
have | simple
|
||||
has | -s form
|
||||
had | past
|
||||
having | -ing form
|
||||
| DO
|
||||
do | simple
|
||||
does | -s form
|
||||
did | past
|
||||
doing | -ing form
|
||||
|
||||
| The forms below are, I believe, best omitted, because of the significant
|
||||
| homonym forms:
|
||||
|
||||
| He made a WILL
|
||||
| old tin CAN
|
||||
| merry month of MAY
|
||||
| a smell of MUST
|
||||
| fight the good fight with all thy MIGHT
|
||||
|
||||
| would, could, should, ought might however be included
|
||||
|
||||
| | AUXILIARIES
|
||||
| | WILL
|
||||
|will
|
||||
|
||||
would
|
||||
|
||||
| | SHALL
|
||||
|shall
|
||||
|
||||
should
|
||||
|
||||
| | CAN
|
||||
|can
|
||||
|
||||
could
|
||||
|
||||
| | MAY
|
||||
|may
|
||||
|might
|
||||
| | MUST
|
||||
|must
|
||||
| | OUGHT
|
||||
|
||||
ought
|
||||
|
||||
| COMPOUND FORMS, increasingly encountered nowadays in 'formal' writing
|
||||
| pronoun + verb
|
||||
|
||||
i'm
|
||||
you're
|
||||
he's
|
||||
she's
|
||||
it's
|
||||
we're
|
||||
they're
|
||||
i've
|
||||
you've
|
||||
we've
|
||||
they've
|
||||
i'd
|
||||
you'd
|
||||
he'd
|
||||
she'd
|
||||
we'd
|
||||
they'd
|
||||
i'll
|
||||
you'll
|
||||
he'll
|
||||
she'll
|
||||
we'll
|
||||
they'll
|
||||
|
||||
| verb + negation
|
||||
|
||||
isn't
|
||||
aren't
|
||||
wasn't
|
||||
weren't
|
||||
hasn't
|
||||
haven't
|
||||
hadn't
|
||||
doesn't
|
||||
don't
|
||||
didn't
|
||||
|
||||
| auxiliary + negation
|
||||
|
||||
won't
|
||||
wouldn't
|
||||
shan't
|
||||
shouldn't
|
||||
can't
|
||||
cannot
|
||||
couldn't
|
||||
mustn't
|
||||
|
||||
| miscellaneous forms
|
||||
|
||||
let's
|
||||
that's
|
||||
who's
|
||||
what's
|
||||
here's
|
||||
there's
|
||||
when's
|
||||
where's
|
||||
why's
|
||||
how's
|
||||
|
||||
| rarer forms
|
||||
|
||||
| daren't needn't
|
||||
|
||||
| doubtful forms
|
||||
|
||||
| oughtn't mightn't
|
||||
|
||||
| ARTICLES
|
||||
a
|
||||
an
|
||||
the
|
||||
|
||||
| THE REST (Overlap among prepositions, conjunctions, adverbs etc is so
|
||||
| high, that classification is pointless.)
|
||||
and
|
||||
but
|
||||
if
|
||||
or
|
||||
because
|
||||
as
|
||||
until
|
||||
while
|
||||
|
||||
of
|
||||
at
|
||||
by
|
||||
for
|
||||
with
|
||||
about
|
||||
against
|
||||
between
|
||||
into
|
||||
through
|
||||
during
|
||||
before
|
||||
after
|
||||
above
|
||||
below
|
||||
to
|
||||
from
|
||||
up
|
||||
down
|
||||
in
|
||||
out
|
||||
on
|
||||
off
|
||||
over
|
||||
under
|
||||
|
||||
again
|
||||
further
|
||||
then
|
||||
once
|
||||
|
||||
here
|
||||
there
|
||||
when
|
||||
where
|
||||
why
|
||||
how
|
||||
|
||||
all
|
||||
any
|
||||
both
|
||||
each
|
||||
few
|
||||
more
|
||||
most
|
||||
other
|
||||
some
|
||||
such
|
||||
|
||||
no
|
||||
nor
|
||||
not
|
||||
only
|
||||
own
|
||||
same
|
||||
so
|
||||
than
|
||||
too
|
||||
very
|
||||
|
||||
| Just for the record, the following words are among the commonest in English
|
||||
|
||||
| one
|
||||
| every
|
||||
| least
|
||||
| less
|
||||
| many
|
||||
| now
|
||||
| ever
|
||||
| never
|
||||
| say
|
||||
| says
|
||||
| said
|
||||
| also
|
||||
| get
|
||||
| go
|
||||
| goes
|
||||
| just
|
||||
| made
|
||||
| make
|
||||
| put
|
||||
| see
|
||||
| seen
|
||||
| whether
|
||||
| like
|
||||
| well
|
||||
| back
|
||||
| even
|
||||
| still
|
||||
| way
|
||||
| take
|
||||
| since
|
||||
| another
|
||||
| however
|
||||
| two
|
||||
| three
|
||||
| four
|
||||
| five
|
||||
| first
|
||||
| second
|
||||
| new
|
||||
| old
|
||||
| high
|
||||
| long
|
||||
@@ -1,3 +1,6 @@
|
||||
# automatically generated by utils/generate-snowball.py
|
||||
# from https://github.com/snowballstem/snowball-website/raw/efb4ae4d65769fb1652acbe608c0c817e746c730/algorithms/spanish/stop.txt
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
SPANISH_STOPWORDS = frozenset({
|
||||
|
||||
@@ -1,4 +1,13 @@
|
||||
|source: https://snowballstem.org/algorithms/spanish/stop.txt
|
||||
|
||||
| A Spanish stop word list. Comments begin with vertical bar. Each stop
|
||||
| word is at the start of a line.
|
||||
|
||||
|
||||
| The following is a ranked list (commonest to rarest) of stopwords
|
||||
| deriving from a large sample of text.
|
||||
|
||||
| Extra words have been added at the end.
|
||||
|
||||
de | from, of
|
||||
la | the, her
|
||||
que | who, that
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
# automatically generated by utils/generate-snowball.py
|
||||
# from https://github.com/snowballstem/snowball-website/raw/efb4ae4d65769fb1652acbe608c0c817e746c730/algorithms/finnish/stop.txt
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
FINNISH_STOPWORDS = frozenset({
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
| source: https://snowballstem.org/algorithms/finnish/stop.txt
|
||||
|
||||
| forms of BE
|
||||
|
||||
olla
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
# automatically generated by utils/generate-snowball.py
|
||||
# from https://github.com/snowballstem/snowball-website/raw/efb4ae4d65769fb1652acbe608c0c817e746c730/algorithms/french/stop.txt
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
FRENCH_STOPWORDS = frozenset({
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
| source: https://snowballstem.org/algorithms/french/stop.txt
|
||||
|
||||
| A French stop word list. Comments begin with vertical bar. Each stop
|
||||
| word is at the start of a line.
|
||||
|
||||
au | a + le
|
||||
aux | a + les
|
||||
avec | with
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
# automatically generated by utils/generate-snowball.py
|
||||
# from https://github.com/snowballstem/snowball-website/raw/efb4ae4d65769fb1652acbe608c0c817e746c730/algorithms/hungarian/stop.txt
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
HUNGARIAN_STOPWORDS = frozenset({
|
||||
@@ -56,11 +59,11 @@ HUNGARIAN_STOPWORDS = frozenset({
|
||||
'ekkor',
|
||||
'el',
|
||||
'ellen',
|
||||
'első',
|
||||
'elsõ',
|
||||
'elég',
|
||||
'elő',
|
||||
'először',
|
||||
'előtt',
|
||||
'elõ',
|
||||
'elõször',
|
||||
'elõtt',
|
||||
'emilyen',
|
||||
'ennek',
|
||||
'erre',
|
||||
@@ -191,12 +194,12 @@ HUNGARIAN_STOPWORDS = frozenset({
|
||||
'éppen',
|
||||
'és',
|
||||
'így',
|
||||
'õ',
|
||||
'õk',
|
||||
'õket',
|
||||
'össze',
|
||||
'úgy',
|
||||
'új',
|
||||
'újabb',
|
||||
'újra',
|
||||
'ő',
|
||||
'ők',
|
||||
'őket',
|
||||
})
|
||||
@@ -1,5 +1,7 @@
|
||||
| source: https://snowballstem.org/algorithms/hungarian/stop.txt
|
||||
|
||||
| Hungarian stop word list
|
||||
| prepared by Anna Tordai
|
||||
|
||||
a
|
||||
ahogy
|
||||
ahol
|
||||
@@ -57,10 +59,10 @@ ekkor
|
||||
el
|
||||
elég
|
||||
ellen
|
||||
elő
|
||||
először
|
||||
előtt
|
||||
első
|
||||
elõ
|
||||
elõször
|
||||
elõtt
|
||||
elsõ
|
||||
én
|
||||
éppen
|
||||
ebben
|
||||
@@ -151,9 +153,9 @@ nincs
|
||||
olyan
|
||||
ott
|
||||
össze
|
||||
ő
|
||||
ők
|
||||
őket
|
||||
õ
|
||||
õk
|
||||
õket
|
||||
pedig
|
||||
persze
|
||||
rá
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
# automatically generated by utils/generate-snowball.py
|
||||
# from https://github.com/snowballstem/snowball-website/raw/efb4ae4d65769fb1652acbe608c0c817e746c730/algorithms/italian/stop.txt
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
ITALIAN_STOPWORDS = frozenset({
|
||||
@@ -213,6 +216,7 @@ ITALIAN_STOPWORDS = frozenset({
|
||||
'sono',
|
||||
'sta',
|
||||
'stai',
|
||||
'stando',
|
||||
'stanno',
|
||||
'starai',
|
||||
'staranno',
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
| source: https://snowballstem.org/algorithms/italian/stop.txt
|
||||
|
||||
| An Italian stop word list. Comments begin with vertical bar. Each stop
|
||||
| word is at the start of a line.
|
||||
|
||||
ad | a (to) before vowel
|
||||
al | a + il
|
||||
allo | a + lo
|
||||
@@ -289,3 +292,4 @@ stessi
|
||||
stesse
|
||||
stessimo
|
||||
stessero
|
||||
stando
|
||||
@@ -1,3 +1,6 @@
|
||||
# automatically generated by utils/generate-snowball.py
|
||||
# from https://github.com/snowballstem/snowball-website/raw/efb4ae4d65769fb1652acbe608c0c817e746c730/algorithms/dutch/stop.txt
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
DUTCH_STOPWORDS = frozenset({
|
||||
|
||||
@@ -1,4 +1,14 @@
|
||||
| source: https://snowballstem.org/algorithms/dutch/stop.txt
|
||||
|
||||
|
||||
| A Dutch stop word list. Comments begin with vertical bar. Each stop
|
||||
| word is at the start of a line.
|
||||
|
||||
| This is a ranked list (commonest to rarest) of stopwords derived from
|
||||
| a large sample of Dutch text.
|
||||
|
||||
| Dutch stop words frequently exhibit homonym clashes. These are indicated
|
||||
| clearly below.
|
||||
|
||||
de | the
|
||||
en | and
|
||||
van | of, from
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
# automatically generated by utils/generate-snowball.py
|
||||
# from https://github.com/snowballstem/snowball-website/raw/efb4ae4d65769fb1652acbe608c0c817e746c730/algorithms/norwegian/stop.txt
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
NORWEGIAN_STOPWORDS = frozenset({
|
||||
|
||||
@@ -1,4 +1,12 @@
|
||||
| source: https://snowballstem.org/algorithms/norwegian/stop.txt
|
||||
|
||||
| A Norwegian stop word list. Comments begin with vertical bar. Each stop
|
||||
| word is at the start of a line.
|
||||
|
||||
| This stop word list is for the dominant bokmål dialect. Words unique
|
||||
| to nynorsk are marked *.
|
||||
|
||||
| Revised by Jan Bruusgaard <Jan.Bruusgaard@ssb.no>, Jan 2005
|
||||
|
||||
og | and
|
||||
i | in
|
||||
jeg | I
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
# automatically generated by utils/generate-snowball.py
|
||||
# from https://github.com/snowballstem/snowball-website/raw/efb4ae4d65769fb1652acbe608c0c817e746c730/algorithms/portuguese/stop.txt
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
PORTUGUESE_STOPWORDS = frozenset({
|
||||
|
||||
@@ -1,4 +1,13 @@
|
||||
| source: https://snowballstem.org/algorithms/portuguese/stop.txt
|
||||
|
||||
| A Portuguese stop word list. Comments begin with vertical bar. Each stop
|
||||
| word is at the start of a line.
|
||||
|
||||
|
||||
| The following is a ranked list (commonest to rarest) of stopwords
|
||||
| deriving from a large sample of text.
|
||||
|
||||
| Extra words have been added at the end.
|
||||
|
||||
de | of, from
|
||||
a | the; to, at; her
|
||||
o | the; him
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
# automatically generated by utils/generate-snowball.py
|
||||
# from https://github.com/snowballstem/snowball-website/raw/efb4ae4d65769fb1652acbe608c0c817e746c730/algorithms/russian/stop.txt
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
RUSSIAN_STOPWORDS = frozenset({
|
||||
|
||||
@@ -1,4 +1,13 @@
|
||||
| source: https://snowballstem.org/algorithms/russian/stop.txt
|
||||
|
||||
|
||||
| a russian stop word list. comments begin with vertical bar. each stop
|
||||
| word is at the start of a line.
|
||||
|
||||
| this is a ranked list (commonest to rarest) of stopwords derived from
|
||||
| a large text sample.
|
||||
|
||||
| letter `ё' is translated to `е'.
|
||||
|
||||
и | and
|
||||
в | in/into
|
||||
во | alternative form
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
# automatically generated by utils/generate-snowball.py
|
||||
# from https://github.com/snowballstem/snowball-website/raw/efb4ae4d65769fb1652acbe608c0c817e746c730/algorithms/swedish/stop.txt
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
SWEDISH_STOPWORDS = frozenset({
|
||||
|
||||
@@ -1,4 +1,13 @@
|
||||
| source: https://snowballstem.org/algorithms/swedish/stop.txt
|
||||
|
||||
| A Swedish stop word list. Comments begin with vertical bar. Each stop
|
||||
| word is at the start of a line.
|
||||
|
||||
| This is a ranked list (commonest to rarest) of stopwords derived from
|
||||
| a large text sample.
|
||||
|
||||
| Swedish stop words occasionally exhibit homonym clashes. For example
|
||||
| så = so, but also seed. These are indicated clearly below.
|
||||
|
||||
och | and
|
||||
det | it, this/that
|
||||
att | to (with infinitive)
|
||||
|
||||
@@ -55,7 +55,14 @@ def regenerate_stopwords() -> None:
|
||||
STOPWORDS_DIR.joinpath('__init__.py').touch()
|
||||
|
||||
for lang_code, lang_name, url in STOPWORD_URLS:
|
||||
data = requests.get(url, timeout=5).content.decode('utf-8')
|
||||
content = requests.get(url, timeout=5).content
|
||||
try:
|
||||
data = content.decode('utf-8')
|
||||
except UnicodeDecodeError:
|
||||
if lang_code == 'ru':
|
||||
data = content.decode('koi8-r')
|
||||
else:
|
||||
data = content.decode('latin-1')
|
||||
|
||||
# record the original source of the stopwords list
|
||||
txt_path = STOPWORDS_DIR / f'{lang_code}.txt'
|
||||
|
||||
Reference in new issue
Block a user