intermediateView Python source

Make thumbnails from a folder of images

This example creates 24 test images (PNG and JPEG, 320 to 1280 pixels wide). It writes a 64-pixel thumbnail for each image and returns a table of metadata. The image bytes stay inside the task that reads them.

Run it

Run all commands below from the repository root (sail-examples/):

uv run multimodal/image-thumbnails/main.py

Output

The newest capture appears first. Older output stays here when a result changes.

Current output · Sail 0.7.3

Captured on Sail 0.7.3. Latest capture: 2026-11-05, Python 3.12.14.

+------+------+-------------+---------------+----------------+
|format|images|source_pixels|max_thumb_width|max_thumb_height|
+------+------+-------------+---------------+----------------+
|  JPEG|    12|      5374528|             64|              64|
|   PNG|    12|      7142400|             64|              48|
+------+------+-------------+---------------+----------------+

+------------+-----+------+-----------+------------+
|        name|width|height|thumb_width|thumb_height|
+------------+-----+------+-----------+------------+
|image-00.png|  320|   240|         64|          48|
|image-01.jpg|  640|   480|         64|          48|
|image-02.png|  800|   600|         64|          48|
|image-03.jpg| 1024|   768|         64|          48|
|image-04.png| 1280|   960|         64|          48|
+------------+-----+------+-----------+------------+
only showing top 5 rows

Python code

main.py143 lines

View main.py on GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143# /// script
# requires-python = ">=3.12"
# dependencies = [
#   "pysail==0.7.3",
#   "pyspark-client==4.1.3",
#   "pyarrow==26.0.0",
#   "pandas==3.0.6",
#   "pillow==13.0.0",
# ]
# ///
"""Make thumbnails for a folder of images: one row per file, small results out.

The pattern: list the files into a table of paths, open each file inside `mapInArrow`, write
the thumbnail, and return only metadata. The image bytes never travel between stages.
"""

import atexit
import random
import shutil
import tempfile
import zlib
from pathlib import Path

import pyarrow as pa
from pyspark.sql import DataFrame, SparkSession
from pyspark.sql import functions as F

THUMBNAIL_SIZE = 64
IMAGE_SUFFIXES = {".jpg", ".jpeg", ".png", ".webp", ".tif", ".tiff"}
SIZES = [(320, 240), (640, 480), (800, 600), (1024, 768), (1280, 960), (500, 500)]

# The declared schema and the Arrow schema of every returned batch must match exactly:
# Apache Spark rejects an int64 column declared as `int`.
COLUMNS = [
    ("name", "string", pa.string()),
    ("format", "string", pa.string()),
    ("width", "int", pa.int32()),
    ("height", "int", pa.int32()),
    ("thumb_width", "int", pa.int32()),
    ("thumb_height", "int", pa.int32()),
    ("mean_r", "int", pa.int32()),
    ("mean_g", "int", pa.int32()),
    ("mean_b", "int", pa.int32()),
    ("thumb_crc32", "long", pa.int64()),
]
SCHEMA = ", ".join(f"{name} {sql}" for name, sql, _ in COLUMNS)
ARROW_SCHEMA = pa.schema([(name, arrow) for name, _, arrow in COLUMNS])


def temp_dir(prefix: str) -> str:
    """A temporary directory that is removed when the process exits."""
    path = tempfile.mkdtemp(prefix=prefix)
    atexit.register(shutil.rmtree, path, ignore_errors=True)
    return path


def make_images(directory: Path, count: int = 24) -> None:
    """Draw `count` deterministic test images, half PNG and half JPEG."""
    from PIL import Image, ImageDraw

    for i in range(count):
        rng = random.Random(i)
        width, height = SIZES[i % len(SIZES)]
        image = Image.new("RGB", (width, height), tuple(rng.randrange(256) for _ in range(3)))
        draw = ImageDraw.Draw(image)
        for _ in range(8):
            x0, y0 = rng.randrange(width), rng.randrange(height)
            x1, y1 = min(width, x0 + rng.randrange(20, width)), min(height, y0 + rng.randrange(20, height))
            color = tuple(rng.randrange(256) for _ in range(3))
            (draw.ellipse if rng.random() < 0.5 else draw.rectangle)([x0, y0, x1, y1], fill=color)
        if i % 2:
            image.save(directory / f"image-{i:02d}.jpg", quality=90)
        else:
            image.save(directory / f"image-{i:02d}.png")


def make_thumbnailer(output_dir: str):
    """Build the mapInArrow function; it writes thumbnails and returns one metadata row per file."""

    def thumbnails(batches):
        from PIL import Image, ImageOps, ImageStat

        for batch in batches:
            rows = {name: [] for name in ARROW_SCHEMA.names}
            for path in batch.column("path").to_pylist():
                with Image.open(path) as image:
                    rows["format"].append(image.format)
                    rows["width"].append(image.width)
                    rows["height"].append(image.height)
                    # Apply the camera's orientation tag, so photos are not stored sideways.
                    thumb = ImageOps.exif_transpose(image).convert("RGB")
                thumb.thumbnail((THUMBNAIL_SIZE, THUMBNAIL_SIZE))
                # Keep the extension in the name: a.jpg and a.png must not overwrite each other.
                thumb.save(Path(output_dir) / f"{Path(path).name}.png")
                mean = [round(c) for c in ImageStat.Stat(thumb).mean]
                rows["name"].append(Path(path).name)
                rows["thumb_width"].append(thumb.width)
                rows["thumb_height"].append(thumb.height)
                rows["mean_r"].append(mean[0])
                rows["mean_g"].append(mean[1])
                rows["mean_b"].append(mean[2])
                rows["thumb_crc32"].append(zlib.crc32(thumb.tobytes()))
            yield pa.RecordBatch.from_pydict(rows, schema=ARROW_SCHEMA)

    return thumbnails


def run(spark: SparkSession) -> dict[str, DataFrame]:
    images = Path(temp_dir("images-"))
    make_images(images)
    output = temp_dir("thumbnails-")

    # A table of paths, one row per file. Each task opens only the files in its own rows.
    files = sorted(str(p) for p in images.iterdir() if p.suffix.lower() in IMAGE_SUFFIXES)
    paths = spark.createDataFrame([(p,) for p in files], "path string").repartition(4)
    metadata = paths.mapInArrow(make_thumbnailer(output), SCHEMA)

    # Save the metadata once: rereading it below must not decode the images again.
    metadata_path = temp_dir("metadata-")
    metadata.write.mode("overwrite").parquet(metadata_path)
    saved = spark.read.parquet(metadata_path)

    summary = saved.groupBy("format").agg(
        F.count("*").alias("images"),
        F.sum(F.col("width") * F.col("height")).alias("source_pixels"),
        F.max("thumb_width").alias("max_thumb_width"),
        F.max("thumb_height").alias("max_thumb_height"),
    )
    return {"thumbnails": saved.orderBy("name"), "summary": summary.orderBy("format")}


if __name__ == "__main__":
    from pysail.spark import SparkConnectServer

    server = SparkConnectServer()
    server.start()
    _, port = server.listening_address
    spark = SparkSession.builder.remote(f"sc://localhost:{port}").getOrCreate()
    results = run(spark)
    results["summary"].show()
    results["thumbnails"].select("name", "width", "height", "thumb_width", "thumb_height").show(5)
    spark.stop()
    server.stop()

How it works

• A table of paths. The file list becomes a DataFrame with one path row per file, split into four partitions. Each task opens only the files in its own rows.
• Decode and resize in mapInArrow. Pillow opens each file, records its format and size, writes a thumbnail, and computes the thumbnail's mean color and checksum.
• Small results out. The function returns a few numbers per image, never the image bytes. Large binary values never flow between stages, so batches stay small.
• Save once. The metadata is written to Parquet before it is aggregated, so later queries never decode the images again.

The same pattern covers PDFs, audio and video: read the file in the UDF, return text, features or embeddings.

Use your own files

Replace make_images and the files list with your own paths. They must be readable from wherever the UDF runs: local paths on one machine, object storage URLs (read with a client such as fsspec) on a cluster.

Related Sail behavior

• Process media through a table of file paths
• Match Arrow output types to the declared schema

Keep exploring

AI data preparationClassify images with an ONNX model →Label photos and send uncertain predictions for review. AI data preparationSearch photos by description →Find a photo by describing what you want to see.