Coverage for dak/generate_packages_sources2.py: 86%
146 statements
« prev ^ index » next coverage.py v7.6.0, created at 2026-08-03 16:46 +0000
1"""
2Generate Packages/Sources files
4@contact: Debian FTP Master <ftpmaster@debian.org>
5@copyright: 2011 Ansgar Burchardt <ansgar@debian.org>
6@copyright: Based on daklib/lists.py and dak/generate_filelist.py:
7 2009-2011 Torsten Werner <twerner@debian.org>
8@copyright: Based on dak/generate_packages_sources.py:
9 2000, 2001, 2002, 2006 James Troup <james@nocrew.org>
10 2009 Mark Hymers <mhy@debian.org>
11 2010 Joerg Jaspert <joerg@debian.org>
12@license: GNU General Public License version 2 or later
13"""
15# This program is free software; you can redistribute it and/or modify
16# it under the terms of the GNU General Public License as published by
17# the Free Software Foundation; either version 2 of the License, or
18# (at your option) any later version.
20# This program is distributed in the hope that it will be useful,
21# but WITHOUT ANY WARRANTY; without even the implied warranty of
22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
23# GNU General Public License for more details.
25# You should have received a copy of the GNU General Public License
26# along with this program; if not, write to the Free Software
27# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
29import sys
30from typing import Any, NoReturn
32import apt_pkg
33from sqlalchemy import select, sql
36def usage() -> NoReturn:
37 print(
38 """Usage: dak generate-packages-sources2 [OPTIONS]
39Generate the Packages/Sources files
41 -a, --archive=ARCHIVE process suites in ARCHIVE
42 -s, --suite=SUITE process this suite
43 Default: All suites not marked 'untouchable'
44 -f, --force Allow processing of untouchable suites
45 CAREFUL: Only to be used at point release time!
46 -h, --help show this help and exit
48SUITE can be a space separated list, e.g.
49 --suite=unstable testing
50"""
51 )
52 sys.exit()
55#############################################################################
58# Here be dragons.
59_sources_query = R"""
60SELECT
62 (SELECT
63 STRING_AGG(
64 CASE
65 WHEN key = 'Source' THEN E'Package\: '
66 WHEN key = 'Files' AND suite.checksums && array['md5sum'] THEN E'Files\:\n ' || f.md5sum || ' ' || f.size || ' ' || SUBSTRING(f.filename FROM E'/([^/]*)\\Z')
67 WHEN key = 'Files' THEN NULL
68 WHEN key = 'Checksums-Sha1' AND suite.checksums && array['sha1'] THEN E'Checksums-Sha1\:\n ' || f.sha1sum || ' ' || f.size || ' ' || SUBSTRING(f.filename FROM E'/([^/]*)\\Z')
69 WHEN key = 'Checksums-Sha1' THEN NULL
70 WHEN key = 'Checksums-Sha256' AND suite.checksums && array['sha256'] THEN E'Checksums-Sha256\:\n ' || f.sha256sum || ' ' || f.size || ' ' || SUBSTRING(f.filename FROM E'/([^/]*)\\Z')
71 WHEN key = 'Checksums-Sha256' THEN NULL
72 ELSE key || E'\: '
73 END || value, E'\n' ORDER BY mk.ordering, mk.key)
74 FROM
75 source_metadata sm
76 JOIN metadata_keys mk ON mk.key_id = sm.key_id
77 WHERE s.id=sm.src_id
78 )
79 ||
80 CASE
81 WHEN src_associations_full.extra_source THEN E'\nExtra-Source-Only\: yes'
82 ELSE ''
83 END
84 ||
85 E'\nDirectory\: pool/' || :component_name || '/' || SUBSTRING(f.filename FROM E'\\A(.*)/[^/]*\\Z')
86 ||
87 E'\nPriority\: ' || COALESCE(pri.priority, 'optional')
88 ||
89 E'\nSection\: ' || COALESCE(sec.section, 'misc')
90 ||
91 E'\n'
93FROM
95source s
96JOIN src_associations_full ON src_associations_full.suite = :suite AND s.id = src_associations_full.source
97JOIN files f ON s.file=f.id
98JOIN files_archive_map fam
99 ON fam.file_id = f.id
100 AND fam.archive_id = (SELECT archive_id FROM suite WHERE id = :suite)
101 AND fam.component_id = :component
102LEFT JOIN override o ON o.package = s.source
103 AND o.suite = :overridesuite
104 AND o.component = :component
105 AND o.type = :dsc_type
106LEFT JOIN section sec ON o.section = sec.id
107LEFT JOIN priority pri ON o.priority = pri.id
108LEFT JOIN suite on suite.id = :suite
110ORDER BY
111s.source, s.version
112"""
115def generate_sources(suite_id: int, component_id: int) -> tuple[int, list[str]]:
116 global _sources_query
117 from daklib.dakmultiprocessing import PROC_STATUS_SUCCESS
118 from daklib.dbconn import Component, DBConn, OverrideType, Suite
119 from daklib.filewriter import SourcesFileWriter
121 session = DBConn().session()
122 dsc_type = session.execute(
123 select(OverrideType.overridetype_id).filter_by(overridetype="dsc")
124 ).scalar_one()
126 suite = session.get_one(Suite, suite_id)
127 component = session.get_one(Component, component_id)
129 overridesuite_id = suite.get_overridesuite().suite_id
131 writer_args: dict[str, Any] = {
132 "archive": suite.archive.path,
133 "suite": suite.suite_name,
134 "component": component.component_name,
135 }
136 if suite.indices_compression is not None: 136 ↛ 138line 136 didn't jump to line 138 because the condition on line 136 was always true
137 writer_args["compression"] = suite.indices_compression
138 writer = SourcesFileWriter(**writer_args)
139 output = writer.open()
141 # run query and write Sources
142 r = session.execute(
143 sql.text(_sources_query),
144 {
145 "suite": suite_id,
146 "component": component_id,
147 "component_name": component.component_name,
148 "dsc_type": dsc_type,
149 "overridesuite": overridesuite_id,
150 },
151 )
152 for (stanza,) in r:
153 if stanza is None: 153 ↛ 154line 153 didn't jump to line 154 because the condition on line 153 was never true
154 raise Exception(
155 f"Failed to generate stanza for Sources in {suite.suite_name}/{component.component_name}"
156 )
157 print(stanza, file=output)
159 writer.close()
161 message = ["generate sources", suite.suite_name, component.component_name]
162 session.rollback()
163 return (PROC_STATUS_SUCCESS, message)
166#############################################################################
169# Here be large dragons.
170_packages_query = R"""
171WITH
173 tmp AS (
174 SELECT
175 b.id AS binary_id,
176 b.package AS package,
177 b.version AS version,
178 b.architecture AS architecture,
179 b.source AS source_id,
180 s.source AS source,
181 f.filename AS filename,
182 f.size AS size,
183 f.md5sum AS md5sum,
184 f.sha1sum AS sha1sum,
185 f.sha256sum AS sha256sum,
186 (SELECT value FROM binaries_metadata
187 WHERE bin_id = b.id
188 AND key_id = (SELECT key_id FROM metadata_keys WHERE key = 'Priority'))
189 AS fallback_priority,
190 (SELECT value FROM binaries_metadata
191 WHERE bin_id = b.id
192 AND key_id = (SELECT key_id FROM metadata_keys WHERE key = 'Section'))
193 AS fallback_section
194 FROM
195 binaries b
196 JOIN bin_associations ba ON b.id = ba.bin
197 JOIN files f ON f.id = b.file
198 JOIN files_archive_map fam ON f.id = fam.file_id AND fam.archive_id = :archive_id
199 JOIN source s ON b.source = s.id
200 WHERE
201 (b.architecture = :arch_all OR b.architecture = :arch) AND b.type = :type_name
202 AND ba.suite = :suite
203 AND fam.component_id = :component
204 )
206SELECT
207 (SELECT
208 STRING_AGG(key || E'\: ' || value, E'\n' ORDER BY ordering, key)
209 FROM
210 (SELECT key, ordering,
211 CASE WHEN :include_long_description = 'false' AND key = 'Description'
212 THEN SUBSTRING(value FROM E'\\A[^\n]*')
213 ELSE value
214 END AS value
215 FROM
216 binaries_metadata bm
217 JOIN metadata_keys mk ON mk.key_id = bm.key_id
218 WHERE
219 bm.bin_id = tmp.binary_id
220 AND key != ALL (:metadata_skip)
221 ) AS metadata
222 )
223 || COALESCE(E'\n' || (SELECT
224 STRING_AGG(key || E'\: ' || value, E'\n' ORDER BY key)
225 FROM external_overrides eo
226 WHERE
227 eo.package = tmp.package
228 AND eo.suite = :overridesuite AND eo.component = :component
229 ), '')
230 || E'\nSection\: ' || COALESCE(sec.section, tmp.fallback_section, 'misc')
231 || E'\nPriority\: ' || COALESCE(pri.priority, tmp.fallback_priority, 'optional')
232 || E'\nFilename\: pool/' || :component_name || '/' || tmp.filename
233 || E'\nSize\: ' || tmp.size
234 || CASE WHEN suite.checksums && array['md5sum'] THEN E'\nMD5sum\: ' || tmp.md5sum ELSE '' END
235 || CASE WHEN suite.checksums && array['sha1'] THEN E'\nSHA1\: ' || tmp.sha1sum ELSE '' END
236 || CASE WHEN suite.checksums && array['sha256'] THEN E'\nSHA256\: ' || tmp.sha256sum ELSE '' END
237 || E'\n'
239FROM
240 tmp
241 LEFT JOIN override o ON o.package = tmp.package
242 AND o.type = :type_id
243 AND o.suite = :overridesuite
244 AND o.component = :component
245 LEFT JOIN section sec ON sec.id = o.section
246 LEFT JOIN priority pri ON pri.id = o.priority
247 LEFT JOIN suite ON suite.id = :suite
249WHERE
250 (
251 architecture <> :arch_all
252 OR
253 (architecture = :arch_all AND source_id IN (SELECT source_id FROM tmp WHERE architecture <> :arch_all))
254 OR
255 (architecture = :arch_all AND source NOT IN (SELECT DISTINCT source FROM tmp WHERE architecture <> :arch_all))
256 )
258ORDER BY tmp.source, tmp.package, tmp.version
259"""
262def generate_packages(
263 suite_id: int, component_id: int, architecture_id: int, type_name: str
264) -> tuple[int, list[str]]:
265 global _packages_query
266 from daklib.dakmultiprocessing import PROC_STATUS_SUCCESS
267 from daklib.dbconn import Architecture, Component, DBConn, OverrideType, Suite
268 from daklib.filewriter import PackagesFileWriter
270 session = DBConn().session()
271 arch_all_id = session.execute(
272 select(Architecture.arch_id).filter_by(arch_string="all")
273 ).scalar_one()
274 type_id = session.execute(
275 select(OverrideType.overridetype_id).filter_by(overridetype=type_name)
276 ).scalar_one()
278 suite = session.get_one(Suite, suite_id)
279 component = session.get_one(Component, component_id)
280 architecture = session.get_one(Architecture, architecture_id)
282 overridesuite_id = suite.get_overridesuite().suite_id
283 include_long_description = suite.include_long_description
285 # We currently filter out the "Tag" line. They are set by external
286 # overrides and NOT by the maintainer. And actually having it set by
287 # maintainer means we output it twice at the moment -> which breaks
288 # dselect.
289 metadata_skip = ["Section", "Priority", "Tag"]
290 if include_long_description: 290 ↛ 291line 290 didn't jump to line 291 because the condition on line 290 was never true
291 metadata_skip.append("Description-md5")
293 writer_args: dict[str, Any] = {
294 "archive": suite.archive.path,
295 "suite": suite.suite_name,
296 "component": component.component_name,
297 "architecture": architecture.arch_string,
298 "debtype": type_name,
299 }
300 if suite.indices_compression is not None: 300 ↛ 302line 300 didn't jump to line 302 because the condition on line 300 was always true
301 writer_args["compression"] = suite.indices_compression
302 writer = PackagesFileWriter(**writer_args)
303 output = writer.open()
305 r = session.execute(
306 sql.text(_packages_query),
307 {
308 "archive_id": suite.archive.archive_id,
309 "suite": suite_id,
310 "component": component_id,
311 "component_name": component.component_name,
312 "arch": architecture_id,
313 "type_id": type_id,
314 "type_name": type_name,
315 "arch_all": arch_all_id,
316 "overridesuite": overridesuite_id,
317 "metadata_skip": metadata_skip,
318 "include_long_description": "true" if include_long_description else "false",
319 },
320 )
321 for (stanza,) in r:
322 if stanza is None: 322 ↛ 323line 322 didn't jump to line 323 because the condition on line 322 was never true
323 raise Exception(
324 f"Failed to generate stanza for Packages in {suite.suite_name}/{component.component_name} ({architecture.arch_string})"
325 )
326 print(stanza, file=output)
328 writer.close()
330 message = [
331 "generate-packages",
332 suite.suite_name,
333 component.component_name,
334 architecture.arch_string,
335 ]
336 session.rollback()
337 return (PROC_STATUS_SUCCESS, message)
340#############################################################################
343_translations_query = r"""
344WITH
345 override_suite AS
346 (SELECT
347 s.id AS id,
348 COALESCE(os.id, s.id) AS overridesuite_id
349 FROM suite AS s LEFT JOIN suite AS os ON s.overridesuite = os.suite_name)
351SELECT
352 E'Package\: ' || b.package
353 || E'\nDescription-md5\: ' || bm_description_md5.value
354 || E'\nDescription-en\: ' || bm_description.value
355 || E'\n'
356FROM binaries b
357 -- join tables for suite and component
358 JOIN bin_associations ba ON b.id = ba.bin
359 JOIN override_suite os ON os.id = ba.suite
360 JOIN override o ON b.package = o.package AND o.suite = os.overridesuite_id AND o.type = (SELECT id FROM override_type WHERE type = 'deb')
362 -- join tables for Description and Description-md5
363 JOIN binaries_metadata bm_description ON b.id = bm_description.bin_id AND bm_description.key_id = (SELECT key_id FROM metadata_keys WHERE key = 'Description')
364 JOIN binaries_metadata bm_description_md5 ON b.id = bm_description_md5.bin_id AND bm_description_md5.key_id = (SELECT key_id FROM metadata_keys WHERE key = 'Description-md5')
366 -- we want to sort by source name
367 JOIN source s ON b.source = s.id
369WHERE ba.suite = :suite AND o.component = :component
370GROUP BY b.package, bm_description_md5.value, bm_description.value
371ORDER BY MIN(s.source), b.package, bm_description_md5.value
372"""
375def generate_translations(suite_id: int, component_id: int) -> tuple[int, list[str]]:
376 global _translations_query
377 from daklib.dakmultiprocessing import PROC_STATUS_SUCCESS
378 from daklib.dbconn import Component, DBConn, Suite
379 from daklib.filewriter import TranslationFileWriter
381 session = DBConn().session()
382 suite = session.get_one(Suite, suite_id)
383 component = session.get_one(Component, component_id)
385 writer_args: dict[str, Any] = {
386 "archive": suite.archive.path,
387 "suite": suite.suite_name,
388 "component": component.component_name,
389 "language": "en",
390 }
391 if suite.i18n_compression is not None: 391 ↛ 393line 391 didn't jump to line 393 because the condition on line 391 was always true
392 writer_args["compression"] = suite.i18n_compression
393 writer = TranslationFileWriter(**writer_args)
394 output = writer.open()
396 r = session.execute(
397 sql.text(_translations_query), {"suite": suite_id, "component": component_id}
398 )
399 for (stanza,) in r:
400 if stanza is None: 400 ↛ 401line 400 didn't jump to line 401 because the condition on line 400 was never true
401 raise Exception(
402 f"Failed to generate stanza for Translation in {suite.suite_name}/{component.component_name}"
403 )
404 print(stanza, file=output)
406 writer.close()
408 message = ["generate-translations", suite.suite_name, component.component_name]
409 session.rollback()
410 return (PROC_STATUS_SUCCESS, message)
413#############################################################################
416def main() -> None:
417 from daklib import daklog
418 from daklib.config import Config
420 cnf = Config()
422 Arguments = [
423 ("h", "help", "Generate-Packages-Sources::Options::Help"),
424 ("a", "archive", "Generate-Packages-Sources::Options::Archive", "HasArg"),
425 ("s", "suite", "Generate-Packages-Sources::Options::Suite", "HasArg"),
426 ("f", "force", "Generate-Packages-Sources::Options::Force"),
427 ("o", "option", "", "ArbItem"),
428 ]
430 apt_pkg.parse_commandline(cnf.Cnf, Arguments, sys.argv) # type: ignore[attr-defined]
431 try:
432 Options = cnf.subtree("Generate-Packages-Sources::Options")
433 except KeyError:
434 Options = {}
436 if "Help" in Options:
437 usage()
439 from daklib.dakmultiprocessing import (
440 PROC_STATUS_SIGNALRAISED,
441 PROC_STATUS_SUCCESS,
442 DakProcessPool,
443 )
445 pool = DakProcessPool()
447 logger = daklog.Logger("generate-packages-sources2")
449 from daklib.dbconn import Archive, DBConn, Suite, get_suite
451 session = DBConn().session()
452 session.execute(sql.text("SELECT add_missing_description_md5()"))
453 session.commit()
455 import daklib.utils
457 if "Suite" in Options:
458 suites: list[Suite] = []
459 suite_names = daklib.utils.split_args(Options["Suite"])
460 for name in suite_names:
461 suite = get_suite(name.lower(), session)
462 if suite: 462 ↛ 465line 462 didn't jump to line 465 because the condition on line 462 was always true
463 suites.append(suite)
464 else:
465 print("I: Cannot find suite %s" % name)
466 logger.log(["Cannot find suite %s" % name])
467 else:
468 query = select(Suite).where(~Suite.untouchable)
469 if "Archive" in Options: 469 ↛ 474line 469 didn't jump to line 474 because the condition on line 469 was always true
470 archive_names = daklib.utils.split_args(Options["Archive"])
471 query = query.join(Suite.archive).where(
472 Archive.archive_name.in_(archive_names)
473 )
474 suites = list(session.scalars(query))
476 force = "Force" in Options and Options["Force"]
478 def parse_results(message):
479 # Split out into (code, msg)
480 code, msg = message
481 if code == PROC_STATUS_SUCCESS: 481 ↛ 483line 481 didn't jump to line 483 because the condition on line 481 was always true
482 logger.log(msg)
483 elif code == PROC_STATUS_SIGNALRAISED:
484 logger.log(["E: Subprocess received signal ", msg])
485 else:
486 logger.log(["E: ", msg])
488 # Lock tables so that nobody can change things underneath us
489 session.execute(sql.text("LOCK TABLE src_associations IN SHARE MODE"))
490 session.execute(sql.text("LOCK TABLE bin_associations IN SHARE MODE"))
492 for s in suites:
493 component_ids = [c.component_id for c in s.components]
494 if s.untouchable and not force: 494 ↛ 495line 494 didn't jump to line 495 because the condition on line 494 was never true
495 import daklib.utils
497 daklib.utils.fubar(
498 "Refusing to touch %s (untouchable and not forced)" % s.suite_name
499 )
500 for c in component_ids:
501 pool.apply_async(generate_sources, [s.suite_id, c], callback=parse_results)
502 if not s.include_long_description: 502 ↛ 506line 502 didn't jump to line 506 because the condition on line 502 was always true
503 pool.apply_async(
504 generate_translations, [s.suite_id, c], callback=parse_results
505 )
506 for a in s.architectures:
507 if a == "source":
508 continue
509 pool.apply_async(
510 generate_packages,
511 [s.suite_id, c, a.arch_id, "deb"],
512 callback=parse_results,
513 )
514 pool.apply_async(
515 generate_packages,
516 [s.suite_id, c, a.arch_id, "udeb"],
517 callback=parse_results,
518 )
520 pool.close()
521 pool.join()
523 # this script doesn't change the database
524 session.close()
526 logger.close()
528 sys.exit(pool.overall_status())