Skip to content

gh-133158: Adjust c-analyzer max_sizes for typeobject.c #133159

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 8 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 24 additions & 2 deletions Tools/c-analyzer/c_parser/parser/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -208,7 +208,29 @@ def _iter_source(lines, *, maxtext=11_000, maxlines=200, showtext=False):
return
# At this point either the file ended prematurely
# or there's "too much" text.
filename, lno, text = srcinfo.filename, srcinfo._start, srcinfo.text
filename, lno_from, lno_to = srcinfo.filename, srcinfo.start, srcinfo.end
text = srcinfo.text
if len(text) > 500:
text = text[:500] + '...'
raise Exception(f'unmatched text ({filename} starting at line {lno}):\n{text}')

if srcinfo.too_much_text(maxtext):
import textwrap
msg = f'''
too much text, try to increase MAX_SIZES[MAXTEXT] in cpython/_parser.py
{filename} starting at line {lno_from} to {lno_to}
has code with length {len(text)} greater than {maxtext}:
{text}
'''
raise RuntimeError(textwrap.dedent(msg))

if srcinfo.too_much_lines(maxlines):
import textwrap
msg = f'''
too many lines, try to increase MAX_SIZES[MAXLINES] in cpython/_parser.py
{filename} starting at line {lno_from} to {lno_to}
has code with number of lines {lno_to - lno_from} greater than {maxlines}:
{text}
'''
raise RuntimeError(textwrap.dedent(msg))

raise RuntimeError(f'unmatched text ({filename} starting at line {lno_from}):\n{text}')
10 changes: 8 additions & 2 deletions Tools/c-analyzer/c_parser/parser/_info.py
Original file line number Diff line number Diff line change
Expand Up @@ -123,10 +123,16 @@ def resolve(self, kind, data, name, parent=None):
def done(self):
self._set_ready()

def too_much_text(self, maxtext):
return maxtext and len(self.text) > maxtext

def too_much_lines(self, maxlines):
return maxlines and self.end - self.start > maxlines

def too_much(self, maxtext, maxlines):
if maxtext and len(self.text) > maxtext:
if self.too_much_text(maxtext):
pass
elif maxlines and self.end - self.start > maxlines:
elif self.too_much_lines(maxlines):
pass
else:
return False
Expand Down
2 changes: 1 addition & 1 deletion Tools/c-analyzer/cpython/_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -324,7 +324,7 @@ def clean_lines(text):
_abs('Modules/_testcapimodule.c'): (20_000, 400),
_abs('Modules/expat/expat.h'): (10_000, 400),
_abs('Objects/stringlib/unicode_format.h'): (10_000, 400),
_abs('Objects/typeobject.c'): (35_000, 200),
_abs('Objects/typeobject.c'): (380_000, 13_000),
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

380k characters is huge. Are there really lines that are that long?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is not a line length. It is a length of piece of code that was analyzed in the moment. It may be a 1/3 of file size, for example. typeobject.c has 385_327 bytes length now, so I decided to set this value to near of file size.

_abs('Python/compile.c'): (20_000, 500),
_abs('Python/optimizer.c'): (100_000, 5_000),
_abs('Python/parking_lot.c'): (40_000, 1000),
Expand Down
Loading