Run a script with a ready Spark session

sail spark run -f script.py (or -f - to read the script from stdin) starts a temporary Sail server, runs the script with a ready spark session, and stops the server. A WARN ... aborting N task(s) line at exit is harmless.

Check targets: pysail==0.7.3 with pyspark-client==4.1.3

The Python assertions below check this behavior. Details attributed to external documentation are outside those assertions. Newer releases may behave differently.

Run the check

Follow the test setup, then run from the repository root (sail-examples/). The harness applies the configuration declared in this check.

Sail

uv run pytest 'tests/test_facts.py::test_fact[sail-spark-run[sail]]'

Python code

sail-spark-run.py40 lines

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40FACT = {
    "id": "sail-spark-run",
    "title": "Run a script with a ready Spark session",
    "category": "run-and-connect",
    "text": """`sail spark run -f script.py` (or `-f -` to read the script from stdin) starts a temporary
        Sail server, runs the script with a ready `spark` session, and stops the server. A
        `WARN ... aborting N task(s)` line at exit is harmless.""",
    "engines": ["sail"],
    "order": 2,
}

import subprocess
import sys
import tempfile
from pathlib import Path


def check(spark, engine):
    script = Path(tempfile.mkdtemp()) / "job.py"
    script.write_text("print('rows', spark.range(5).count())\n")
    sail = Path(sys.executable).parent / "sail"
    proc = subprocess.run(
        [str(sail), "spark", "run", "-f", str(script)],
        capture_output=True,
        text=True,
        timeout=120,
        check=False,
    )
    assert proc.returncode == 0, f"exit {proc.returncode}: {proc.stderr[-500:]}"
    assert "rows 5" in proc.stdout, f"unexpected output: {proc.stdout[-300:]}"
    stdin = subprocess.run(
        [str(sail), "spark", "run", "-f", "-"],
        input=script.read_text(),
        capture_output=True,
        text=True,
        timeout=120,
        check=False,
    )
    assert stdin.returncode == 0 and "rows 5" in stdin.stdout, f"-f - failed: {stdin.stderr[-300:]}"
    return {"exit": proc.returncode, "warn_at_exit": "aborting" in proc.stderr}