anteater v2
This commit is contained in:
Hubert Van De Walle
parent
b4b0a9f71e
commit
5b92d2a8a5
2
.gitignore
vendored
Normal file
2
.gitignore
vendored
Normal file
@ -0,0 +1,2 @@
|
||||
/.idea
|
||||
__pycache__/
|
||||
BIN
anteater.jpg
Normal file
BIN
anteater.jpg
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 644 KiB |
17
git.py
Normal file
17
git.py
Normal file
@ -0,0 +1,17 @@
|
||||
import subprocess
|
||||
|
||||
|
||||
class GitRepository:
|
||||
def __init__(self, path):
|
||||
self.path = path
|
||||
|
||||
def checkout(self, hash):
|
||||
subprocess.check_call(["git", "-C", self.path, "checkout", hash])
|
||||
|
||||
def between(self, old_hash, new_hash):
|
||||
res = subprocess.run(
|
||||
["git", "-C", self.path, "log", "--oneline", f"{old_hash}..{new_hash}"],
|
||||
capture_output=True,
|
||||
check=True,
|
||||
)
|
||||
return res.stdout.decode("utf-8").rstrip().split("\n")
|
||||
181
main.py
181
main.py
@ -1,90 +1,101 @@
|
||||
import click
|
||||
import re
|
||||
import requests
|
||||
import subprocess
|
||||
import os
|
||||
import subprocess
|
||||
import threading
|
||||
from shutil import rmtree
|
||||
from bs4 import BeautifulSoup
|
||||
from time import sleep
|
||||
from typing import Optional
|
||||
|
||||
worktree_root = "/home/hubert/src/18.0-proj"
|
||||
import click
|
||||
from rich import print as rprint
|
||||
from rich.live import Live
|
||||
from rich.panel import Panel
|
||||
|
||||
bundles = {
|
||||
"17.0": "17-0-192736",
|
||||
"18.0": "18-0-320432",
|
||||
from runbot import _commits_from_batch, _get_batches, Repo, bundles
|
||||
from git import GitRepository
|
||||
|
||||
worktrees = {
|
||||
"17.0": "/home/hubert/src/17.0-proj",
|
||||
"18.0": "/home/hubert/src/18.0-proj",
|
||||
}
|
||||
|
||||
|
||||
def _get_bundle_last_page(version: str) -> int:
|
||||
bundle = bundles[version]
|
||||
url = f"https://runbot.odoo.com/runbot/bundle/{bundle}/page/1000"
|
||||
res = requests.get(url)
|
||||
assert res.status_code == 200
|
||||
soup = BeautifulSoup(res.text, "html.parser")
|
||||
text = soup.select_one(".page-item.active").get_text(strip=True)
|
||||
return int(text)
|
||||
|
||||
|
||||
def _commits_from_batch(id: int) -> dict:
|
||||
url = f"https://runbot.odoo.com/runbot/batch/{id}"
|
||||
res = requests.get(url)
|
||||
assert res.status_code == 200
|
||||
soup = BeautifulSoup(res.text, "html.parser")
|
||||
commit_nodes = soup.select("a[title='View Commit on Github']")
|
||||
commits = dict()
|
||||
assert len(commit_nodes) > 0
|
||||
gh_re = re.compile(r"https://github.com/odoo/([\w-]+)/commit/(\w+)")
|
||||
for commit in commit_nodes:
|
||||
match = gh_re.search(commit["href"])
|
||||
repo = match.group(1)
|
||||
commit = match.group(2)
|
||||
commits[repo] = commit
|
||||
|
||||
assert "odoo" in commits
|
||||
assert "enterprise" in commits
|
||||
return commits
|
||||
|
||||
|
||||
def _checkout_commits(commits: dict):
|
||||
subprocess.check_call(
|
||||
["git", "-C", f"{worktree_root}/odoo", "checkout", commits["odoo"]]
|
||||
)
|
||||
subprocess.check_call(
|
||||
["git", "-C", f"{worktree_root}/enterprise", "checkout", commits["enterprise"]]
|
||||
)
|
||||
|
||||
|
||||
def _run_odoo(test_tags: str) -> bool:
|
||||
def _run_odoo(worktree_root, test_tags: str) -> bool:
|
||||
venv_path = f"{worktree_root}/venv-bisect"
|
||||
rmtree(venv_path, ignore_errors=True)
|
||||
subprocess.check_call(["uv", "venv", "-p", "3.12", venv_path])
|
||||
subprocess.check_call(
|
||||
["uv", "venv", "-p", "3.12", venv_path],
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
)
|
||||
env = os.environ.copy()
|
||||
env["VIRTUAL_ENV"] = venv_path
|
||||
env["PYTHONUNBUFFERED"] = "1"
|
||||
subprocess.check_call(
|
||||
["uv", "pip", "install", "-r", f"{worktree_root}/odoo/requirements.txt"],
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
env=env,
|
||||
)
|
||||
res = subprocess.run(
|
||||
subprocess.check_call(
|
||||
["uv", "pip", "install", "websocket-client", "mock", "dbfread"],
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
env=env,
|
||||
)
|
||||
process = subprocess.Popen(
|
||||
["odoo", "--no-patch", "--drop", "--test-tags", test_tags],
|
||||
cwd=worktree_root,
|
||||
env=env,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
bufsize=1,
|
||||
)
|
||||
return res.returncode == 0
|
||||
|
||||
limit = 10
|
||||
|
||||
def _get_batches(bundle, page=1) -> list[int]:
|
||||
limit = 3000
|
||||
url = f"https://runbot.odoo.com/runbot/bundle/{bundle}?limit={limit}"
|
||||
res = requests.get(url)
|
||||
assert res.status_code == 200
|
||||
soup = BeautifulSoup(res.text, "html.parser")
|
||||
batch_nodes = soup.select("div.batch_row")
|
||||
batch_url_re = re.compile(r"^/runbot/batch/(\d+)$")
|
||||
ids = []
|
||||
for node in batch_nodes:
|
||||
link = node.select_one("a[title='View Batch']")["href"]
|
||||
match = batch_url_re.search(link)
|
||||
ids.append(match.group(1))
|
||||
return ids
|
||||
output_lines = []
|
||||
|
||||
def read_stream(stream, container):
|
||||
for line in stream:
|
||||
container.append(line.rstrip())
|
||||
|
||||
stdout_thread = threading.Thread(
|
||||
target=read_stream, args=(process.stdout, output_lines)
|
||||
)
|
||||
stdout_thread.start()
|
||||
|
||||
with Live(refresh_per_second=10) as live:
|
||||
while process.poll() is None:
|
||||
display_lines = output_lines[-limit:]
|
||||
panel = Panel(
|
||||
"\n".join(display_lines), title="Command Output", border_style="green"
|
||||
)
|
||||
live.update(panel)
|
||||
sleep(0.1)
|
||||
|
||||
stdout_thread.join()
|
||||
|
||||
sleep(0.2)
|
||||
|
||||
if process.returncode == 0:
|
||||
display_lines = output_lines[-limit:]
|
||||
extra = ""
|
||||
if len(output_lines) > limit:
|
||||
extra = f"\n[dim]... (showing last {limit} of {len(output_lines)} lines)[/dim]"
|
||||
panel = Panel(
|
||||
"\n".join(display_lines) + extra, title="Success", border_style="green"
|
||||
)
|
||||
else:
|
||||
full_output = "\n".join(output_lines)
|
||||
panel = Panel(
|
||||
f"[bold red]Command failed (exit {process.returncode})[/bold red]\n\n{full_output}",
|
||||
title="Failure",
|
||||
border_style="red",
|
||||
)
|
||||
live.update(panel, refresh=True)
|
||||
|
||||
return process.returncode == 0
|
||||
|
||||
|
||||
def _find_first_failing_commit(batches: list[int], test_failed):
|
||||
@ -103,25 +114,53 @@ def _find_first_failing_commit(batches: list[int], test_failed):
|
||||
raise ValueError("404")
|
||||
|
||||
|
||||
def _bisect(bundle: str, test_tags: str):
|
||||
def _bisect(version: str, test_tags: str) -> Optional[dict[Repo, str]]:
|
||||
bundle = bundles[version]
|
||||
worktree_root = worktrees[version]
|
||||
odoo_repository = GitRepository(f"{worktree_root}/odoo")
|
||||
enterprise_repository = GitRepository(f"{worktree_root}/enterprise")
|
||||
batches = list(reversed(_get_batches(bundle)))
|
||||
|
||||
def test_failed(batch: int):
|
||||
commits = _commits_from_batch(batch)
|
||||
_checkout_commits(commits)
|
||||
odoo_repository.checkout(commits["odoo"])
|
||||
enterprise_repository.checkout(commits["enterprise"])
|
||||
return not _run_odoo(test_tags)
|
||||
|
||||
suspect = _find_first_failing_commit(batches, test_failed)
|
||||
if suspect:
|
||||
|
||||
def print_batch(name, batch, commits):
|
||||
rprint(f"[green]{name}")
|
||||
rprint(commits)
|
||||
|
||||
idx = batches.index(suspect)
|
||||
previous_batch = batches[idx - 1]
|
||||
commits = _commits_from_batch(suspect)
|
||||
return commits
|
||||
previous_commits = _commits_from_batch(previous_batch)
|
||||
print_batch("suspect", suspect, commits)
|
||||
print_batch("previous", previous_batch, commits)
|
||||
|
||||
rprint("[blue]odoo")
|
||||
rprint(odoo_repository.between(previous_commits["odoo"], commits["odoo"]))
|
||||
|
||||
rprint("[blue]enterprise")
|
||||
rprint(
|
||||
enterprise_repository.between(
|
||||
previous_commits["enterprise"], commits["enterprise"]
|
||||
)
|
||||
)
|
||||
|
||||
return None # TODO
|
||||
|
||||
return None
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.option("--version", default="18.0", type=click.Choice(bundles.keys()))
|
||||
@click.option("--version", default="17.0", type=click.Choice(bundles.keys()))
|
||||
@click.option("--test-tags")
|
||||
def main(version, test_tags):
|
||||
print(_bisect(bundles["18.0"], test_tags))
|
||||
_bisect(version, test_tags)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@ -8,4 +8,5 @@ dependencies = [
|
||||
"beautifulsoup4>=4.13.4",
|
||||
"click>=8.1.8",
|
||||
"requests>=2.32.3",
|
||||
"rich>=14.0.0",
|
||||
]
|
||||
|
||||
60
runbot.py
Normal file
60
runbot.py
Normal file
@ -0,0 +1,60 @@
|
||||
import re
|
||||
from typing import Literal
|
||||
|
||||
import requests
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
bundles = {
|
||||
"17.0": "17-0-192736",
|
||||
"18.0": "18-0-320432",
|
||||
}
|
||||
|
||||
Repo = Literal[
|
||||
"odoo", "enterprise", "design-themes", "upgrade", "documentation", "upgrade-util"
|
||||
]
|
||||
|
||||
|
||||
def _commits_from_batch(id: int) -> dict[Repo, str]:
|
||||
url = f"https://runbot.odoo.com/runbot/batch/{id}"
|
||||
res = requests.get(url)
|
||||
assert res.status_code == 200
|
||||
soup = BeautifulSoup(res.text, "html.parser")
|
||||
commit_nodes = soup.select("a[title='View Commit on Github']")
|
||||
commits = dict()
|
||||
assert len(commit_nodes) > 0
|
||||
gh_re = re.compile(r"https://github.com/odoo/([\w-]+)/commit/(\w+)")
|
||||
for commit in commit_nodes:
|
||||
match = gh_re.search(commit["href"])
|
||||
repo = match.group(1)
|
||||
commit = match.group(2)
|
||||
commits[repo] = commit
|
||||
|
||||
assert "odoo" in commits
|
||||
assert "enterprise" in commits
|
||||
return commits
|
||||
|
||||
|
||||
def _get_bundle_last_page(version: str) -> int:
|
||||
bundle = bundles[version]
|
||||
url = f"https://runbot.odoo.com/runbot/bundle/{bundle}/page/1000"
|
||||
res = requests.get(url)
|
||||
assert res.status_code == 200
|
||||
soup = BeautifulSoup(res.text, "html.parser")
|
||||
text = soup.select_one(".page-item.active").get_text(strip=True)
|
||||
return int(text)
|
||||
|
||||
|
||||
def _get_batches(bundle, page=1) -> list[int]:
|
||||
limit = 5000
|
||||
url = f"https://runbot.odoo.com/runbot/bundle/{bundle}?limit={limit}"
|
||||
res = requests.get(url)
|
||||
assert res.status_code == 200
|
||||
soup = BeautifulSoup(res.text, "html.parser")
|
||||
batch_nodes = soup.select("div.batch_row")
|
||||
batch_url_re = re.compile(r"^/runbot/batch/(\d+)$")
|
||||
ids = []
|
||||
for node in batch_nodes:
|
||||
link = node.select_one("a[title='View Batch']")["href"]
|
||||
match = batch_url_re.search(link)
|
||||
ids.append(match.group(1))
|
||||
return ids
|
||||
45
uv.lock
generated
45
uv.lock
generated
@ -10,6 +10,7 @@ dependencies = [
|
||||
{ name = "beautifulsoup4" },
|
||||
{ name = "click" },
|
||||
{ name = "requests" },
|
||||
{ name = "rich" },
|
||||
]
|
||||
|
||||
[package.metadata]
|
||||
@ -17,6 +18,7 @@ requires-dist = [
|
||||
{ name = "beautifulsoup4", specifier = ">=4.13.4" },
|
||||
{ name = "click", specifier = ">=8.1.8" },
|
||||
{ name = "requests", specifier = ">=2.32.3" },
|
||||
{ name = "rich", specifier = ">=14.0.0" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@ -106,6 +108,36 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442, upload_time = "2024-09-15T18:07:37.964Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "markdown-it-py"
|
||||
version = "3.0.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "mdurl" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/38/71/3b932df36c1a044d397a1f92d1cf91ee0a503d91e470cbd670aa66b07ed0/markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb", size = 74596, upload_time = "2023-06-03T06:41:14.443Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/42/d7/1ec15b46af6af88f19b8e5ffea08fa375d433c998b8a7639e76935c14f1f/markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1", size = 87528, upload_time = "2023-06-03T06:41:11.019Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mdurl"
|
||||
version = "0.1.2"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload_time = "2022-08-14T12:40:10.846Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload_time = "2022-08-14T12:40:09.779Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pygments"
|
||||
version = "2.19.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/7c/2d/c3338d48ea6cc0feb8446d8e6937e1408088a72a39937982cc6111d17f84/pygments-2.19.1.tar.gz", hash = "sha256:61c16d2a8576dc0649d9f39e089b5f02bcd27fba10d8fb4dcc28173f7a45151f", size = 4968581, upload_time = "2025-01-06T17:26:30.443Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/8a/0b/9fcc47d19c48b59121088dd6da2488a49d5f72dacf8262e2790a1d2c7d15/pygments-2.19.1-py3-none-any.whl", hash = "sha256:9ea1544ad55cecf4b8242fab6dd35a93bbce657034b0611ee383099054ab6d8c", size = 1225293, upload_time = "2025-01-06T17:26:25.553Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "requests"
|
||||
version = "2.32.3"
|
||||
@ -121,6 +153,19 @@ wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/f9/9b/335f9764261e915ed497fcdeb11df5dfd6f7bf257d4a6a2a686d80da4d54/requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6", size = 64928, upload_time = "2024-05-29T15:37:47.027Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rich"
|
||||
version = "14.0.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "markdown-it-py" },
|
||||
{ name = "pygments" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/a1/53/830aa4c3066a8ab0ae9a9955976fb770fe9c6102117c8ec4ab3ea62d89e8/rich-14.0.0.tar.gz", hash = "sha256:82f1bc23a6a21ebca4ae0c45af9bdbc492ed20231dcb63f297d6d1021a9d5725", size = 224078, upload_time = "2025-03-30T14:15:14.23Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/0d/9b/63f4c7ebc259242c89b3acafdb37b41d1185c07ff0011164674e9076b491/rich-14.0.0-py3-none-any.whl", hash = "sha256:1c9491e1951aac09caffd42f448ee3d04e58923ffe14993f6e83068dc395d7e0", size = 243229, upload_time = "2025-03-30T14:15:12.283Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "soupsieve"
|
||||
version = "2.7"
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user