shell bypass 403
UnknownSec Shell
:
/
opt
/
cloudlinux
/
venv
/
lib64
/
python3.11
/
site-packages
/
pylint
/
checkers
/ [
drwxr-xr-x
]
upload
mass deface
mass delete
console
info server
name :
unicode.py
# Licensed under the GPL: https://www.gnu.org/licenses/old-licenses/gpl-2.0.html # For details: https://github.com/PyCQA/pylint/blob/main/LICENSE # Copyright (c) https://github.com/PyCQA/pylint/blob/main/CONTRIBUTORS.txt """Unicode and some other ASCII characters can be used to create programs that run much different compared to what a human reader would expect from them. PEP 672 lists some examples. See: https://www.python.org/dev/peps/pep-0672/ The following checkers are intended to make users are aware of these issues. """ from __future__ import annotations import codecs import contextlib import io import re from collections import OrderedDict from collections.abc import Iterable from functools import lru_cache from tokenize import detect_encoding from typing import NamedTuple, TypeVar from astroid import nodes import pylint.interfaces import pylint.lint from pylint import checkers _StrLike = TypeVar("_StrLike", str, bytes) # Based on: # https://golangexample.com/go-linter-which-checks-for-dangerous-unicode-character-sequences/ # We use '\u' because it doesn't require a map lookup and is therefore faster BIDI_UNICODE = [ "\u202A", # \N{LEFT-TO-RIGHT EMBEDDING} "\u202B", # \N{RIGHT-TO-LEFT EMBEDDING} "\u202C", # \N{POP DIRECTIONAL FORMATTING} "\u202D", # \N{LEFT-TO-RIGHT OVERRIDE} "\u202E", # \N{RIGHT-TO-LEFT OVERRIDE} "\u2066", # \N{LEFT-TO-RIGHT ISOLATE} "\u2067", # \N{RIGHT-TO-LEFT ISOLATE} "\u2068", # \N{FIRST STRONG ISOLATE} "\u2069", # \N{POP DIRECTIONAL ISOLATE} # The following was part of PEP 672: # https://www.python.org/dev/peps/pep-0672/ # so the list above might not be complete "\u200F", # \n{RIGHT-TO-LEFT MARK} # We don't use # "\u200E" # \n{LEFT-TO-RIGHT MARK} # as this is the default for latin files and can't be used # to hide code ] class _BadChar(NamedTuple): """Representation of an ASCII char considered bad.""" name: str unescaped: str escaped: str code: str help_text: str def description(self) -> str: """Used for the detailed error message description.""" return ( f"Invalid unescaped character {self.name}, " f'use "{self.escaped}" instead.' ) def human_code(self) -> str: """Used to generate the human readable error message.""" return f"invalid-character-{self.name}" # Based on https://www.python.org/dev/peps/pep-0672/ BAD_CHARS = [ _BadChar( "backspace", "\b", "\\b", "E2510", ( "Moves the cursor back, so the character after it will overwrite the " "character before." ), ), _BadChar( "carriage-return", "\r", "\\r", "E2511", ( "Moves the cursor to the start of line, subsequent characters overwrite " "the start of the line." ), ), _BadChar( "sub", "\x1A", "\\x1A", "E2512", ( 'Ctrl+Z "End of text" on Windows. Some programs (such as type) ignore ' "the rest of the file after it." ), ), _BadChar( "esc", "\x1B", "\\x1B", "E2513", ( "Commonly initiates escape codes which allow arbitrary control " "of the terminal." ), ), _BadChar( "nul", "\0", "\\0", "E2514", "Mostly end of input for python.", ), _BadChar( # Zero Width with Space. At the time of writing not accepted by Python. # But used in Trojan Source Examples, so still included and tested for. "zero-width-space", "\u200B", # \n{ZERO WIDTH SPACE} "\\u200B", "E2515", "Invisible space character could hide real code execution.", ), ] BAD_ASCII_SEARCH_DICT = {char.unescaped: char for char in BAD_CHARS} def _line_length(line: _StrLike, codec: str) -> int: """Get the length of a string like line as displayed in an editor.""" if isinstance(line, bytes): decoded = _remove_bom(line, codec).decode(codec, "replace") else: decoded = line stripped = decoded.rstrip("\n") if stripped != decoded: stripped = stripped.rstrip("\r") return len(stripped) def _map_positions_to_result( line: _StrLike, search_dict: dict[_StrLike, _BadChar], new_line: _StrLike, byte_str_length: int = 1, ) -> dict[int, _BadChar]: """Get all occurrences of search dict keys within line. Ignores Windows end of line and can handle bytes as well as string. Also takes care of encodings for which the length of an encoded code point does not default to 8 Bit. """ result: dict[int, _BadChar] = {} for search_for, char in search_dict.items(): if search_for not in line: continue # Special Handling for Windows '\r\n' if char.unescaped == "\r" and line.endswith(new_line): ignore_pos = len(line) - 2 * byte_str_length else: ignore_pos = None start = 0 pos = line.find(search_for, start) while pos > 0: if pos != ignore_pos: # Calculate the column col = int(pos / byte_str_length) result[col] = char start = pos + 1 pos = line.find(search_for, start) return result UNICODE_BOMS = { "utf-8": codecs.BOM_UTF8, "utf-16": codecs.BOM_UTF16, "utf-32": codecs.BOM_UTF32, "utf-16le": codecs.BOM_UTF16_LE, "utf-16be": codecs.BOM_UTF16_BE, "utf-32le": codecs.BOM_UTF32_LE, "utf-32be": codecs.BOM_UTF32_BE, } BOM_SORTED_TO_CODEC = OrderedDict( # Sorted by length of BOM of each codec (UNICODE_BOMS[codec], codec) for codec in ("utf-32le", "utf-32be", "utf-8", "utf-16le", "utf-16be") ) UTF_NAME_REGEX_COMPILED = re.compile( "utf[ -]?(8|16|32)[ -]?(le|be|)?(sig)?", flags=re.IGNORECASE ) def _normalize_codec_name(codec: str) -> str: """Make sure the codec name is always given as defined in the BOM dict.""" return UTF_NAME_REGEX_COMPILED.sub(r"utf-\1\2", codec).lower() def _remove_bom(encoded: bytes, encoding: str) -> bytes: """Remove the bom if given from a line.""" if encoding not in UNICODE_BOMS: return encoded bom = UNICODE_BOMS[encoding] if encoded.startswith(bom): return encoded[len(bom) :] return encoded def _encode_without_bom(string: str, encoding: str) -> bytes: """Encode a string but remove the BOM.""" return _remove_bom(string.encode(encoding), encoding) def _byte_to_str_length(codec: str) -> int: """Return how many byte are usually(!) a character point.""" if codec.startswith("utf-32"): return 4 if codec.startswith("utf-16"): return 2 return 1 @lru_cache(maxsize=1000) def _cached_encode_search(string: str, encoding: str) -> bytes: """A cached version of encode used for search pattern.""" return _encode_without_bom(string, encoding) def _fix_utf16_32_line_stream(steam: Iterable[bytes], codec: str) -> Iterable[bytes]: """Handle line ending for UTF16 and UTF32 correctly. Currently, Python simply strips the required zeros after \n after the line ending. Leading to lines that can't be decoded properly """ if not codec.startswith("utf-16") and not codec.startswith("utf-32"): yield from steam else: # First we get all the bytes in memory content = b"".join(line for line in steam) new_line = _cached_encode_search("\n", codec) # Now we split the line by the real new line in the correct encoding # we can't use split as it would strip the \n that we need start = 0 while True: pos = content.find(new_line, start) if pos >= 0: yield content[start : pos + len(new_line)] else: # Yield the rest and finish if content[start:]: yield content[start:] break start = pos + len(new_line) def extract_codec_from_bom(first_line: bytes) -> str: """Try to extract the codec (unicode only) by checking for the BOM. For details about BOM see https://unicode.org/faq/utf_bom.html#BOM Args: first_line: the first line of a file Returns: a codec name Raises: ValueError: if no codec was found """ for bom, codec in BOM_SORTED_TO_CODEC.items(): if first_line.startswith(bom): return codec raise ValueError("No BOM found. Could not detect Unicode codec.") class UnicodeChecker(checkers.BaseRawFileChecker): """Check characters that could be used to hide bad code to humans. This includes: - Bidirectional Unicode (see https://trojansource.codes/) - Bad ASCII characters (see PEP672) If a programmer requires to use such a character they should use the escaped version, that is also much easier to read and does not depend on the editor used. The Checker also includes a check that UTF-16 and UTF-32 are not used to encode Python files. At the time of writing Python supported only UTF-8. See https://stackoverflow.com/questions/69897842/ and https://bugs.python.org/issue1503789 for background. """ name = "unicode_checker" msgs = { "E2501": ( # This error will be only displayed to users once Python Supports # UTF-16/UTF-32 (if at all) "UTF-16 and UTF-32 aren't backward compatible. Use UTF-8 instead", "invalid-unicode-codec", ( "For compatibility use UTF-8 instead of UTF-16/UTF-32. " "See also https://bugs.python.org/issue1503789 for a history " "of this issue. And " "https://softwareengineering.stackexchange.com/questions/102205/ " "for some possible problems when using UTF-16 for instance." ), ), "E2502": ( ( "Contains control characters that can permit obfuscated code " "executed differently than displayed" ), "bidirectional-unicode", ( "bidirectional unicode are typically not displayed characters required " "to display right-to-left (RTL) script " "(i.e. Chinese, Japanese, Arabic, Hebrew, ...) correctly. " "So can you trust this code? " "Are you sure it displayed correctly in all editors? " "If you did not write it or your language is not RTL," " remove the special characters, as they could be used to trick you into " "executing code, " "that does something else than what it looks like.\n" "More Information:\n" "https://en.wikipedia.org/wiki/Bidirectional_text\n" "https://trojansource.codes/" ), ), "C2503": ( "PEP8 recommends UTF-8 as encoding for Python files", "bad-file-encoding", ( "PEP8 recommends UTF-8 default encoding for Python files. See " "https://peps.python.org/pep-0008/#source-file-encoding" ), ), **{ bad_char.code: ( bad_char.description(), bad_char.human_code(), bad_char.help_text, ) for bad_char in BAD_CHARS }, } @staticmethod def _is_invalid_codec(codec: str) -> bool: return codec.startswith("utf-16") or codec.startswith("utf-32") @staticmethod def _is_unicode(codec: str) -> bool: return codec.startswith("utf") @classmethod def _find_line_matches(cls, line: bytes, codec: str) -> dict[int, _BadChar]: """Find all matches of BAD_CHARS within line. Args: line: the input codec: that will be used to convert line/or search string into Return: A dictionary with the column offset and the BadASCIIChar """ # We try to decode in Unicode to get the correct column offset # if we would use bytes, it could be off because UTF-8 has no fixed length try: line_search = line.decode(codec, errors="strict") search_dict = BAD_ASCII_SEARCH_DICT return _map_positions_to_result(line_search, search_dict, "\n") except UnicodeDecodeError: # If we can't decode properly, we simply use bytes, even so the column offsets # might be wrong a bit, but it is still better then nothing line_search_byte = line search_dict_byte: dict[bytes, _BadChar] = {} for char in BAD_CHARS: # Some characters might not exist in all encodings with contextlib.suppress(UnicodeDecodeError): search_dict_byte[ _cached_encode_search(char.unescaped, codec) ] = char return _map_positions_to_result( line_search_byte, search_dict_byte, _cached_encode_search("\n", codec), byte_str_length=_byte_to_str_length(codec), ) @staticmethod def _determine_codec(stream: io.BytesIO) -> tuple[str, int]: """Determine the codec from the given stream. first tries https://www.python.org/dev/peps/pep-0263/ and if this fails also checks for BOMs of UTF-16 and UTF-32 to be future-proof. Args: stream: The byte stream to analyse Returns: A tuple consisting of: - normalized codec name - the line in which the codec was found Raises: SyntaxError: if failing to detect codec """ try: # First try to detect encoding with PEP 263 # Doesn't work with UTF-16/32 at the time of writing # see https://bugs.python.org/issue1503789 codec, lines = detect_encoding(stream.readline) # lines are empty if UTF-8 BOM is found codec_definition_line = len(lines) or 1 except SyntaxError as e: # Codec could not be detected by Python, we try manually to check for # UTF 16/32 BOMs, which aren't supported by Python at the time of writing. # This is only included to be future save and handle these codecs as well stream.seek(0) try: codec = extract_codec_from_bom(stream.readline()) codec_definition_line = 1 except ValueError as ve: # Failed to detect codec, so the syntax error originated not from # UTF16/32 codec usage. So simply raise the error again. raise e from ve return _normalize_codec_name(codec), codec_definition_line def _check_codec(self, codec: str, codec_definition_line: int) -> None: """Check validity of the codec.""" if codec != "utf-8": msg = "bad-file-encoding" if self._is_invalid_codec(codec): msg = "invalid-unicode-codec" self.add_message( msg, # Currently Nodes will lead to crashes of pylint # node=node, line=codec_definition_line, end_lineno=codec_definition_line, confidence=pylint.interfaces.HIGH, col_offset=None, end_col_offset=None, ) def _check_invalid_chars(self, line: bytes, lineno: int, codec: str) -> None: """Look for chars considered bad.""" matches = self._find_line_matches(line, codec) for col, char in matches.items(): self.add_message( char.human_code(), # Currently Nodes will lead to crashes of pylint # node=node, line=lineno, end_lineno=lineno, confidence=pylint.interfaces.HIGH, col_offset=col + 1, end_col_offset=col + len(char.unescaped) + 1, ) def _check_bidi_chars(self, line: bytes, lineno: int, codec: str) -> None: """Look for Bidirectional Unicode, if we use unicode.""" if not self._is_unicode(codec): return for dangerous in BIDI_UNICODE: if _cached_encode_search(dangerous, codec) in line: # Note that we don't add a col_offset on purpose: # Using these unicode characters it depends on the editor # how it displays the location of characters in the line. # So we mark the complete line. self.add_message( "bidirectional-unicode", # Currently Nodes will lead to crashes of pylint # node=node, line=lineno, end_lineno=lineno, # We mark the complete line, as bidi controls make it hard # to determine the correct cursor position within an editor col_offset=0, end_col_offset=_line_length(line, codec), confidence=pylint.interfaces.HIGH, ) # We look for bidirectional unicode only once per line # as we mark the complete line anyway break def process_module(self, node: nodes.Module) -> None: """Perform the actual check by checking module stream.""" with node.stream() as stream: codec, codec_line = self._determine_codec(stream) self._check_codec(codec, codec_line) stream.seek(0) # Check for invalid content (controls/chars) for lineno, line in enumerate( _fix_utf16_32_line_stream(stream, codec), start=1 ): if lineno == 1: line = _remove_bom(line, codec) self._check_bidi_chars(line, lineno, codec) self._check_invalid_chars(line, lineno, codec) def register(linter: pylint.lint.PyLinter) -> None: linter.register_checker(UnicodeChecker(linter))
© 2024 UnknownSec