Skip to content

wfsa

WFSA

Bases: Potential

A weighted finite state automaton (WFSA) potential.

This class wraps a genlm_grammar.WFSA and provides methods for computing the log-weight of a context, the prefix log-weight of a context, and the log-weights of the next token given a context.

Attributes:

Name Type Description
wfsa WFSA

The weighted finite state automaton used for potential calculations.
Source code in genlm/control/potential/built_in/wfsa.py 
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
class WFSA(Potential):
    """
    A weighted finite state automaton (WFSA) potential.

    This class wraps a `genlm_grammar.WFSA` and provides methods for computing the log-weight of a context,
    the prefix log-weight of a context, and the log-weights of the next token given a context.

    Attributes:
        wfsa (genlm_grammar.WFSA): The weighted finite state automaton used for potential calculations.
    """

    def __init__(self, wfsa):
        """
        Initializes the WFSA potential.

        Args:
            wfsa (genlm_grammar.WFSA): The weighted finite state automaton.

        Raises:
            ValueError: If the semiring of the provided WFSA is not Float or Log.

        Note:
            The WFSA will be converted to the Log semiring to avoid underflow if the semiring is Float.
        """
        if wfsa.R not in (Float, Log):
            raise ValueError(f"Unsupported semiring: {wfsa.R}")

        if wfsa.R is Float:
            self.wfsa = self._convert_to_log(wfsa)
        else:
            self.wfsa = wfsa

        self.cache = {(): self.wfsa.epsremove.start}
        super().__init__(vocabulary=list(self.wfsa.alphabet))

    @classmethod
    def from_regex(cls, pattern, charset=None, to_bytes=True):
        """
        Create a WFSA from a regex pattern.

        Args:
            pattern (str): The regex pattern to convert into a WFSA.
            charset (set): The character set to use for negative character classes.
                Defaults to characters in string.printable.
            to_bytes (bool): Whether to convert the WFSA transitions to bytes.
                Defaults to True. When set to False, the WFSA transitions will be strings.

        Returns:
            (WFSA): An instance of the WFSA class.

        Note:
            The transition weights are automatically normalized to form a probability distribution.
            For each state, the weights of all outgoing transitions (including final state transitions)
            sum to 1.0. This means if a state has n possible transitions, each transition will have
            weight 1/n. To create a WFSA from a regex with non-probabilistic transitions, use `BoolFSA`.
        """
        charset = charset or set(string.printable)
        wfsa = interegular_to_wfsa(pattern, charset=charset)
        if to_bytes:
            wfsa = wfsa.to_bytes()
        return cls(wfsa=wfsa)

    @staticmethod
    def _convert_to_log(wfsa):
        """Convert a WFSA from the Float semiring to the Log semiring."""
        assert wfsa.R is Float
        assert isinstance(wfsa, BaseWFSA)
        new = BaseWFSA(Log)

        for i, w in wfsa.I:
            new.add_I(i, Log(np.log(w)))

        for i, w in wfsa.F:
            new.add_F(i, Log(np.log(w)))

        for i, a, j, w in wfsa.arcs():
            new.add_arc(i, a, j, Log(np.log(w)))

        return new

    def _consume(self, bs):
        # XXX implement cache eviction
        bs = tuple(bs)

        try:
            return self.cache[bs]
        except KeyError:
            pass

        wfsa = self.wfsa.epsremove
        curr = wfsa.R.chart()
        prev = self._consume(bs[:-1])
        for i in prev:
            for j, w in wfsa.arcs(i, bs[-1]):
                curr[j] += prev[i] * w

        self.cache[bs] = curr

        return curr

    async def complete(self, context):
        """
        Computes the log weight of the context under the weighted language represented by the WFSA.

        For example, if the WFSA accepts "cat" and "car" with weights $w_{cat}$ and $w_{car}$:\n
        - `complete("c")` returns $-\\infty$ since this sequence is not accepted by the WFSA\n
        - `complete("cat")` returns $\\log(w_{cat})$\n
        - `complete("d")` returns $-\\infty$ since this sequence is not accepted by the WFSA

        Args:
            context (list): A sequence of tokens in the WFSA's alphabet.

        Returns:
            (float): Log weight of context under the WFSA.
        """
        # TODO: optimize to use _consume cache
        return self.wfsa(context).score

    def _prefix(self, context):
        curr = self._consume(context)

        if not curr:
            return float("-inf"), curr

        bkwd = self.wfsa.epsremove.backward
        log_ctx_w = logsumexp([(curr[i] * bkwd[i]).score for i in curr])

        if np.isnan(log_ctx_w):
            return float("-inf"), curr

        return log_ctx_w, curr

    async def prefix(self, context):
        """
        Computes the prefix log weight of `context` under the WFSA.

        This corresponds to the log of the sum of the weights of all sequences with prefix `context`.

        For example, if the WFSA accepts "cat" and "car" with weights $w_{cat}$ and $w_{car}$:\n
        - `prefix("c")` returns $\\log(w_{cat} + w_{car})$\n
        - `prefix("ca")` returns $\\log(w_{cat})$\n
        - `prefix("d")` returns $-\\infty$ since the WFSA does not accept any sequences with prefix "d"

        Args:
            context (list): A sequence of tokens in the WFSA's alphabet.

        Returns:
            (float): Log weight of `context` as a prefix under the WFSA.
        """
        return self._prefix(context)[0]

    async def logw_next(self, context):
        """Returns next token log weights given `context`.

        Args:
            context (list): A sequence of tokens in the WFSA's alphabet.

        Returns:
            (LazyWeights): Log-weights for next token and EOS.
        """
        log_ctx_w, curr = self._prefix(context)

        if log_ctx_w == float("-inf"):
            raise ValueError(f"Context {context!r} has zero weight.")

        bkwd = self.wfsa.epsremove.backward

        ws = self.wfsa.R.chart()
        for i in curr:
            for b, j, w in self.wfsa.epsremove.arcs(i=i):
                ws[b] += curr[i] * w * bkwd[j]

        ws[self.eos] = self.wfsa.R.zero
        for j, w in self.wfsa.epsremove.F:
            ws[self.eos] += curr[j] * w

        log_ws = np.array([ws[b].score for b in self.vocab_eos]) - log_ctx_w

        return self.make_lazy_weights(log_ws)

    def _repr_svg_(self):
        return self.wfsa._repr_svg_()

    def __repr__(self):
        return f"WFSA(wfsa={self.wfsa!r})"

    def spawn(self):
        cls = type(self)
        return cls(wfsa=self.wfsa)

    def clear_cache(self):
        self.cache = {(): self.wfsa.epsremove.start}

__init__(wfsa)

Initializes the WFSA potential.

Parameters:

Name Type Description Default
wfsa WFSA

The weighted finite state automaton.

 required

Raises:

Type Description
ValueError

If the semiring of the provided WFSA is not Float or Log.
Note

The WFSA will be converted to the Log semiring to avoid underflow if the semiring is Float.

Source code in genlm/control/potential/built_in/wfsa.py 
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
def __init__(self, wfsa):
    """
    Initializes the WFSA potential.

    Args:
        wfsa (genlm_grammar.WFSA): The weighted finite state automaton.

    Raises:
        ValueError: If the semiring of the provided WFSA is not Float or Log.

    Note:
        The WFSA will be converted to the Log semiring to avoid underflow if the semiring is Float.
    """
    if wfsa.R not in (Float, Log):
        raise ValueError(f"Unsupported semiring: {wfsa.R}")

    if wfsa.R is Float:
        self.wfsa = self._convert_to_log(wfsa)
    else:
        self.wfsa = wfsa

    self.cache = {(): self.wfsa.epsremove.start}
    super().__init__(vocabulary=list(self.wfsa.alphabet))

from_regex(pattern, charset=None, to_bytes=True) classmethod

Create a WFSA from a regex pattern.

Parameters:

Name Type Description Default
pattern str

The regex pattern to convert into a WFSA.

 required
charset set

The character set to use for negative character classes. Defaults to characters in string.printable.

 None
to_bytes bool

Whether to convert the WFSA transitions to bytes. Defaults to True. When set to False, the WFSA transitions will be strings.

 True

Returns:

Type Description
WFSA

An instance of the WFSA class.
Note

The transition weights are automatically normalized to form a probability distribution. For each state, the weights of all outgoing transitions (including final state transitions) sum to 1.0. This means if a state has n possible transitions, each transition will have weight 1/n. To create a WFSA from a regex with non-probabilistic transitions, use BoolFSA.

Source code in genlm/control/potential/built_in/wfsa.py 
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
@classmethod
def from_regex(cls, pattern, charset=None, to_bytes=True):
    """
    Create a WFSA from a regex pattern.

    Args:
        pattern (str): The regex pattern to convert into a WFSA.
        charset (set): The character set to use for negative character classes.
            Defaults to characters in string.printable.
        to_bytes (bool): Whether to convert the WFSA transitions to bytes.
            Defaults to True. When set to False, the WFSA transitions will be strings.

    Returns:
        (WFSA): An instance of the WFSA class.

    Note:
        The transition weights are automatically normalized to form a probability distribution.
        For each state, the weights of all outgoing transitions (including final state transitions)
        sum to 1.0. This means if a state has n possible transitions, each transition will have
        weight 1/n. To create a WFSA from a regex with non-probabilistic transitions, use `BoolFSA`.
    """
    charset = charset or set(string.printable)
    wfsa = interegular_to_wfsa(pattern, charset=charset)
    if to_bytes:
        wfsa = wfsa.to_bytes()
    return cls(wfsa=wfsa)

complete(context) async

Computes the log weight of the context under the weighted language represented by the WFSA.

For example, if the WFSA accepts "cat" and "car" with weights \(w_{cat}\) and \(w_{car}\):

  • complete("c") returns \(-\infty\) since this sequence is not accepted by the WFSA

  • complete("cat") returns \(\log(w_{cat})\)

  • complete("d") returns \(-\infty\) since this sequence is not accepted by the WFSA

Parameters:

Name Type Description Default
context list

A sequence of tokens in the WFSA's alphabet.

 required

Returns:

Type Description
float

Log weight of context under the WFSA.
Source code in genlm/control/potential/built_in/wfsa.py 
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
async def complete(self, context):
    """
    Computes the log weight of the context under the weighted language represented by the WFSA.

    For example, if the WFSA accepts "cat" and "car" with weights $w_{cat}$ and $w_{car}$:\n
    - `complete("c")` returns $-\\infty$ since this sequence is not accepted by the WFSA\n
    - `complete("cat")` returns $\\log(w_{cat})$\n
    - `complete("d")` returns $-\\infty$ since this sequence is not accepted by the WFSA

    Args:
        context (list): A sequence of tokens in the WFSA's alphabet.

    Returns:
        (float): Log weight of context under the WFSA.
    """
    # TODO: optimize to use _consume cache
    return self.wfsa(context).score

prefix(context) async

Computes the prefix log weight of context under the WFSA.

This corresponds to the log of the sum of the weights of all sequences with prefix context.

For example, if the WFSA accepts "cat" and "car" with weights \(w_{cat}\) and \(w_{car}\):

  • prefix("c") returns \(\log(w_{cat} + w_{car})\)

  • prefix("ca") returns \(\log(w_{cat})\)

  • prefix("d") returns \(-\infty\) since the WFSA does not accept any sequences with prefix "d"

Parameters:

Name Type Description Default
context list

A sequence of tokens in the WFSA's alphabet.

 required

Returns:

Type Description
float

Log weight of context as a prefix under the WFSA.
Source code in genlm/control/potential/built_in/wfsa.py 
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
async def prefix(self, context):
    """
    Computes the prefix log weight of `context` under the WFSA.

    This corresponds to the log of the sum of the weights of all sequences with prefix `context`.

    For example, if the WFSA accepts "cat" and "car" with weights $w_{cat}$ and $w_{car}$:\n
    - `prefix("c")` returns $\\log(w_{cat} + w_{car})$\n
    - `prefix("ca")` returns $\\log(w_{cat})$\n
    - `prefix("d")` returns $-\\infty$ since the WFSA does not accept any sequences with prefix "d"

    Args:
        context (list): A sequence of tokens in the WFSA's alphabet.

    Returns:
        (float): Log weight of `context` as a prefix under the WFSA.
    """
    return self._prefix(context)[0]

logw_next(context) async

Returns next token log weights given context.

Parameters:

Name Type Description Default
context list

A sequence of tokens in the WFSA's alphabet.

 required

Returns:

Type Description
LazyWeights

Log-weights for next token and EOS.
Source code in genlm/control/potential/built_in/wfsa.py 
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
async def logw_next(self, context):
    """Returns next token log weights given `context`.

    Args:
        context (list): A sequence of tokens in the WFSA's alphabet.

    Returns:
        (LazyWeights): Log-weights for next token and EOS.
    """
    log_ctx_w, curr = self._prefix(context)

    if log_ctx_w == float("-inf"):
        raise ValueError(f"Context {context!r} has zero weight.")

    bkwd = self.wfsa.epsremove.backward

    ws = self.wfsa.R.chart()
    for i in curr:
        for b, j, w in self.wfsa.epsremove.arcs(i=i):
            ws[b] += curr[i] * w * bkwd[j]

    ws[self.eos] = self.wfsa.R.zero
    for j, w in self.wfsa.epsremove.F:
        ws[self.eos] += curr[j] * w

    log_ws = np.array([ws[b].score for b in self.vocab_eos]) - log_ctx_w

    return self.make_lazy_weights(log_ws)

BoolFSA

Bases: WFSA

Boolean FSA potential. Returns 0 for accepted, -inf for rejected.

from_regex constructs a Boolean-semiring WFSA by default; the semiring="log" path is deprecated (the Log-semiring FSA closure is unsound because Log.star is partial).

Source code in genlm/control/potential/built_in/wfsa.py 
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
class BoolFSA(WFSA):
    """Boolean FSA potential. Returns ``0`` for accepted, ``-inf`` for rejected.

    ``from_regex`` constructs a Boolean-semiring WFSA by default; the
    ``semiring="log"`` path is deprecated (the Log-semiring FSA closure is
    unsound because ``Log.star`` is partial).
    """

    def __init__(self, wfsa):
        """Initialize the BoolFSA from a WFSA.

        Args:
            wfsa (genlm_grammar.WFSA): A Float, Log, or Boolean WFSA.
                Float is converted to Log; Boolean is kept as-is.
        """
        if wfsa.R is Boolean:
            self.wfsa = wfsa
            self.cache = {(): self.wfsa.epsremove.start}
            Potential.__init__(self, vocabulary=list(self.wfsa.alphabet))
        else:
            super().__init__(wfsa)

    @classmethod
    def from_regex(cls, pattern, charset=None, to_bytes=True, semiring="boolean"):
        """Create a BoolFSA from a regex pattern.

        Args:
            pattern (str): regex pattern.
            charset (set): see ``WFSA.from_regex``.
            to_bytes (bool): see ``WFSA.from_regex``.
            semiring (str): ``"boolean"`` (default) or ``"log"``. ``"log"``
                is deprecated; its closure is unsound (``Log.star`` is partial).

        Returns:
            (BoolFSA): An instance of the BoolFSA class.
        """
        if semiring not in ("log", "boolean"):
            raise ValueError(
                f"semiring must be 'log' or 'boolean', got {semiring!r}"
            )
        if semiring == "log":
            warnings.warn(
                "semiring='log' is deprecated: the FSA closure is unsound on "
                "the Log semiring because Log.star is partial. "
                "Use semiring='boolean'.",
                DeprecationWarning,
                stacklevel=2,
            )
        charset = charset or set(string.printable)
        wfsa = interegular_to_wfsa(pattern, charset=charset)
        if to_bytes:
            wfsa = wfsa.to_bytes()
        if semiring == "boolean":
            wfsa = _float_to_boolean(wfsa)
        return cls(wfsa)

    def _prefix(self, context):
        if self.wfsa.R is Boolean:
            curr = self._consume(context)
            if not curr:
                return float("-inf"), curr
            bkwd = self.wfsa.epsremove.backward
            for i in curr:
                if bkwd[i].score:  # i is co-accessible
                    return 0.0, curr
            return float("-inf"), curr
        return super()._prefix(context)

    async def prefix(self, context):
        """Tests whether `context` is accepted as a prefix.

        Args:
            context (list): A sequence of tokens in the WFSA's alphabet.

        Returns:
            (float): ``0`` if accepted as a prefix, ``-inf`` otherwise.
        """
        if self.wfsa.R is Boolean:
            return self._prefix(context)[0]
        prefix_w = await super().prefix(context)
        if prefix_w > float("-inf"):
            return 0
        return float("-inf")

    async def complete(self, context):
        """Tests whether `context` is accepted.

        Args:
            context (list): A sequence of tokens in the WFSA's alphabet.

        Returns:
            (float): ``0`` if accepted, ``-inf`` otherwise.
        """
        if self.wfsa.R is Boolean:
            curr = self._consume(context)
            if not curr:
                return float("-inf")
            finals = dict(self.wfsa.epsremove.F)
            for i in curr:
                if i in finals and finals[i].score:
                    return 0.0
            return float("-inf")
        complete_w = await super().complete(context)
        if complete_w > float("-inf"):
            return 0
        return float("-inf")

    async def logw_next(self, context):
        """Per-token admissibility log weights given `context`.

        Args:
            context (list): A sequence of tokens in the WFSA's alphabet.

        Returns:
            (LazyWeights): ``0`` for admissible next tokens (incl. EOS),
                ``-inf`` for rejected.
        """
        if self.wfsa.R is Boolean:
            curr = self._consume(context)
            if not curr:
                raise ValueError(f"Context {context!r} has zero weight.")
            bkwd = self.wfsa.epsremove.backward
            finals = dict(self.wfsa.epsremove.F)
            out = self.alloc_logws()
            for i in curr:
                for b, j, w in self.wfsa.epsremove.arcs(i=i):
                    if w.score and bkwd[j].score and b in self.lookup:
                        out[self.lookup[b]] = 0.0
            for i in curr:
                if i in finals and finals[i].score:
                    out[self.lookup[self.eos]] = 0.0
                    break
            return self.make_lazy_weights(out)
        logw_next = await super().logw_next(context)
        return logw_next.spawn(
            new_weights=np.where(
                logw_next.weights > float("-inf"), 0, logw_next.weights
            )
        )

    async def batch_logw_next(self, contexts):
        """Per-token admissibility log weights for a batch of contexts.

        Args:
            contexts (list): The list of contexts.

        Returns:
            (list): List of ``LazyWeights``, one per context.
        """
        if self.wfsa.R is Boolean:
            return [await self.logw_next(c) for c in contexts]
        logw_nexts = await super().batch_logw_next(contexts)
        return [
            logw_next.spawn(
                new_weights=np.where(
                    logw_next.weights > float("-inf"), 0, logw_next.weights
                )
            )
            for logw_next in logw_nexts
        ]

    def __repr__(self):
        return f"BoolFSA(wfsa={self.wfsa!r})"

__init__(wfsa)

Initialize the BoolFSA from a WFSA.

Parameters:

Name Type Description Default
wfsa WFSA

A Float, Log, or Boolean WFSA. Float is converted to Log; Boolean is kept as-is.

 required
Source code in genlm/control/potential/built_in/wfsa.py 
233
234
235
236
237
238
239
240
241
242
243
244
245
def __init__(self, wfsa):
    """Initialize the BoolFSA from a WFSA.

    Args:
        wfsa (genlm_grammar.WFSA): A Float, Log, or Boolean WFSA.
            Float is converted to Log; Boolean is kept as-is.
    """
    if wfsa.R is Boolean:
        self.wfsa = wfsa
        self.cache = {(): self.wfsa.epsremove.start}
        Potential.__init__(self, vocabulary=list(self.wfsa.alphabet))
    else:
        super().__init__(wfsa)

from_regex(pattern, charset=None, to_bytes=True, semiring='boolean') classmethod

Create a BoolFSA from a regex pattern.

Parameters:

Name Type Description Default
pattern str

regex pattern.

 required
charset set

see WFSA.from_regex.

 None
to_bytes bool

see WFSA.from_regex.

 True
semiring str

"boolean" (default) or "log". "log" is deprecated; its closure is unsound (Log.star is partial).

 'boolean'

Returns:

Type Description
BoolFSA

An instance of the BoolFSA class.
Source code in genlm/control/potential/built_in/wfsa.py 
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
@classmethod
def from_regex(cls, pattern, charset=None, to_bytes=True, semiring="boolean"):
    """Create a BoolFSA from a regex pattern.

    Args:
        pattern (str): regex pattern.
        charset (set): see ``WFSA.from_regex``.
        to_bytes (bool): see ``WFSA.from_regex``.
        semiring (str): ``"boolean"`` (default) or ``"log"``. ``"log"``
            is deprecated; its closure is unsound (``Log.star`` is partial).

    Returns:
        (BoolFSA): An instance of the BoolFSA class.
    """
    if semiring not in ("log", "boolean"):
        raise ValueError(
            f"semiring must be 'log' or 'boolean', got {semiring!r}"
        )
    if semiring == "log":
        warnings.warn(
            "semiring='log' is deprecated: the FSA closure is unsound on "
            "the Log semiring because Log.star is partial. "
            "Use semiring='boolean'.",
            DeprecationWarning,
            stacklevel=2,
        )
    charset = charset or set(string.printable)
    wfsa = interegular_to_wfsa(pattern, charset=charset)
    if to_bytes:
        wfsa = wfsa.to_bytes()
    if semiring == "boolean":
        wfsa = _float_to_boolean(wfsa)
    return cls(wfsa)

prefix(context) async

Tests whether context is accepted as a prefix.

Parameters:

Name Type Description Default
context list

A sequence of tokens in the WFSA's alphabet.

 required

Returns:

Type Description
float

0 if accepted as a prefix, -inf otherwise.
Source code in genlm/control/potential/built_in/wfsa.py 
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
async def prefix(self, context):
    """Tests whether `context` is accepted as a prefix.

    Args:
        context (list): A sequence of tokens in the WFSA's alphabet.

    Returns:
        (float): ``0`` if accepted as a prefix, ``-inf`` otherwise.
    """
    if self.wfsa.R is Boolean:
        return self._prefix(context)[0]
    prefix_w = await super().prefix(context)
    if prefix_w > float("-inf"):
        return 0
    return float("-inf")

complete(context) async

Tests whether context is accepted.

Parameters:

Name Type Description Default
context list

A sequence of tokens in the WFSA's alphabet.

 required

Returns:

Type Description
float

0 if accepted, -inf otherwise.
Source code in genlm/control/potential/built_in/wfsa.py 
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
async def complete(self, context):
    """Tests whether `context` is accepted.

    Args:
        context (list): A sequence of tokens in the WFSA's alphabet.

    Returns:
        (float): ``0`` if accepted, ``-inf`` otherwise.
    """
    if self.wfsa.R is Boolean:
        curr = self._consume(context)
        if not curr:
            return float("-inf")
        finals = dict(self.wfsa.epsremove.F)
        for i in curr:
            if i in finals and finals[i].score:
                return 0.0
        return float("-inf")
    complete_w = await super().complete(context)
    if complete_w > float("-inf"):
        return 0
    return float("-inf")

logw_next(context) async

Per-token admissibility log weights given context.

Parameters:

Name Type Description Default
context list

A sequence of tokens in the WFSA's alphabet.

 required

Returns:

Type Description
LazyWeights

0 for admissible next tokens (incl. EOS), -inf for rejected.
Source code in genlm/control/potential/built_in/wfsa.py 
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
async def logw_next(self, context):
    """Per-token admissibility log weights given `context`.

    Args:
        context (list): A sequence of tokens in the WFSA's alphabet.

    Returns:
        (LazyWeights): ``0`` for admissible next tokens (incl. EOS),
            ``-inf`` for rejected.
    """
    if self.wfsa.R is Boolean:
        curr = self._consume(context)
        if not curr:
            raise ValueError(f"Context {context!r} has zero weight.")
        bkwd = self.wfsa.epsremove.backward
        finals = dict(self.wfsa.epsremove.F)
        out = self.alloc_logws()
        for i in curr:
            for b, j, w in self.wfsa.epsremove.arcs(i=i):
                if w.score and bkwd[j].score and b in self.lookup:
                    out[self.lookup[b]] = 0.0
        for i in curr:
            if i in finals and finals[i].score:
                out[self.lookup[self.eos]] = 0.0
                break
        return self.make_lazy_weights(out)
    logw_next = await super().logw_next(context)
    return logw_next.spawn(
        new_weights=np.where(
            logw_next.weights > float("-inf"), 0, logw_next.weights
        )
    )

batch_logw_next(contexts) async

Per-token admissibility log weights for a batch of contexts.

Parameters:

Name Type Description Default
contexts list

The list of contexts.

 required

Returns:

Type Description
list

List of LazyWeights, one per context.
Source code in genlm/control/potential/built_in/wfsa.py 
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
async def batch_logw_next(self, contexts):
    """Per-token admissibility log weights for a batch of contexts.

    Args:
        contexts (list): The list of contexts.

    Returns:
        (list): List of ``LazyWeights``, one per context.
    """
    if self.wfsa.R is Boolean:
        return [await self.logw_next(c) for c in contexts]
    logw_nexts = await super().batch_logw_next(contexts)
    return [
        logw_next.spawn(
            new_weights=np.where(
                logw_next.weights > float("-inf"), 0, logw_next.weights
            )
        )
        for logw_next in logw_nexts
    ]