ยปCore Development>Code coverage>Lib/nntplib.py

Python code coverage for Lib/nntplib.py

#countcontent
1n/a"""An NNTP client class based on:
2n/a- RFC 977: Network News Transfer Protocol
3n/a- RFC 2980: Common NNTP Extensions
4n/a- RFC 3977: Network News Transfer Protocol (version 2)
5n/a
6n/aExample:
7n/a
8n/a>>> from nntplib import NNTP
9n/a>>> s = NNTP('news')
10n/a>>> resp, count, first, last, name = s.group('comp.lang.python')
11n/a>>> print('Group', name, 'has', count, 'articles, range', first, 'to', last)
12n/aGroup comp.lang.python has 51 articles, range 5770 to 5821
13n/a>>> resp, subs = s.xhdr('subject', '{0}-{1}'.format(first, last))
14n/a>>> resp = s.quit()
15n/a>>>
16n/a
17n/aHere 'resp' is the server response line.
18n/aError responses are turned into exceptions.
19n/a
20n/aTo post an article from a file:
21n/a>>> f = open(filename, 'rb') # file containing article, including header
22n/a>>> resp = s.post(f)
23n/a>>>
24n/a
25n/aFor descriptions of all methods, read the comments in the code below.
26n/aNote that all arguments and return values representing article numbers
27n/aare strings, not numbers, since they are rarely used for calculations.
28n/a"""
29n/a
30n/a# RFC 977 by Brian Kantor and Phil Lapsley.
31n/a# xover, xgtitle, xpath, date methods by Kevan Heydon
32n/a
33n/a# Incompatible changes from the 2.x nntplib:
34n/a# - all commands are encoded as UTF-8 data (using the "surrogateescape"
35n/a# error handler), except for raw message data (POST, IHAVE)
36n/a# - all responses are decoded as UTF-8 data (using the "surrogateescape"
37n/a# error handler), except for raw message data (ARTICLE, HEAD, BODY)
38n/a# - the `file` argument to various methods is keyword-only
39n/a#
40n/a# - NNTP.date() returns a datetime object
41n/a# - NNTP.newgroups() and NNTP.newnews() take a datetime (or date) object,
42n/a# rather than a pair of (date, time) strings.
43n/a# - NNTP.newgroups() and NNTP.list() return a list of GroupInfo named tuples
44n/a# - NNTP.descriptions() returns a dict mapping group names to descriptions
45n/a# - NNTP.xover() returns a list of dicts mapping field names (header or metadata)
46n/a# to field values; each dict representing a message overview.
47n/a# - NNTP.article(), NNTP.head() and NNTP.body() return a (response, ArticleInfo)
48n/a# tuple.
49n/a# - the "internal" methods have been marked private (they now start with
50n/a# an underscore)
51n/a
52n/a# Other changes from the 2.x/3.1 nntplib:
53n/a# - automatic querying of capabilities at connect
54n/a# - New method NNTP.getcapabilities()
55n/a# - New method NNTP.over()
56n/a# - New helper function decode_header()
57n/a# - NNTP.post() and NNTP.ihave() accept file objects, bytes-like objects and
58n/a# arbitrary iterables yielding lines.
59n/a# - An extensive test suite :-)
60n/a
61n/a# TODO:
62n/a# - return structured data (GroupInfo etc.) everywhere
63n/a# - support HDR
64n/a
65n/a# Imports
66n/aimport re
67n/aimport socket
68n/aimport collections
69n/aimport datetime
70n/aimport warnings
71n/a
72n/atry:
73n/a import ssl
74n/aexcept ImportError:
75n/a _have_ssl = False
76n/aelse:
77n/a _have_ssl = True
78n/a
79n/afrom email.header import decode_header as _email_decode_header
80n/afrom socket import _GLOBAL_DEFAULT_TIMEOUT
81n/a
82n/a__all__ = ["NNTP",
83n/a "NNTPError", "NNTPReplyError", "NNTPTemporaryError",
84n/a "NNTPPermanentError", "NNTPProtocolError", "NNTPDataError",
85n/a "decode_header",
86n/a ]
87n/a
88n/a# maximal line length when calling readline(). This is to prevent
89n/a# reading arbitrary length lines. RFC 3977 limits NNTP line length to
90n/a# 512 characters, including CRLF. We have selected 2048 just to be on
91n/a# the safe side.
92n/a_MAXLINE = 2048
93n/a
94n/a
95n/a# Exceptions raised when an error or invalid response is received
96n/aclass NNTPError(Exception):
97n/a """Base class for all nntplib exceptions"""
98n/a def __init__(self, *args):
99n/a Exception.__init__(self, *args)
100n/a try:
101n/a self.response = args[0]
102n/a except IndexError:
103n/a self.response = 'No response given'
104n/a
105n/aclass NNTPReplyError(NNTPError):
106n/a """Unexpected [123]xx reply"""
107n/a pass
108n/a
109n/aclass NNTPTemporaryError(NNTPError):
110n/a """4xx errors"""
111n/a pass
112n/a
113n/aclass NNTPPermanentError(NNTPError):
114n/a """5xx errors"""
115n/a pass
116n/a
117n/aclass NNTPProtocolError(NNTPError):
118n/a """Response does not begin with [1-5]"""
119n/a pass
120n/a
121n/aclass NNTPDataError(NNTPError):
122n/a """Error in response data"""
123n/a pass
124n/a
125n/a
126n/a# Standard port used by NNTP servers
127n/aNNTP_PORT = 119
128n/aNNTP_SSL_PORT = 563
129n/a
130n/a# Response numbers that are followed by additional text (e.g. article)
131n/a_LONGRESP = {
132n/a '100', # HELP
133n/a '101', # CAPABILITIES
134n/a '211', # LISTGROUP (also not multi-line with GROUP)
135n/a '215', # LIST
136n/a '220', # ARTICLE
137n/a '221', # HEAD, XHDR
138n/a '222', # BODY
139n/a '224', # OVER, XOVER
140n/a '225', # HDR
141n/a '230', # NEWNEWS
142n/a '231', # NEWGROUPS
143n/a '282', # XGTITLE
144n/a}
145n/a
146n/a# Default decoded value for LIST OVERVIEW.FMT if not supported
147n/a_DEFAULT_OVERVIEW_FMT = [
148n/a "subject", "from", "date", "message-id", "references", ":bytes", ":lines"]
149n/a
150n/a# Alternative names allowed in LIST OVERVIEW.FMT response
151n/a_OVERVIEW_FMT_ALTERNATIVES = {
152n/a 'bytes': ':bytes',
153n/a 'lines': ':lines',
154n/a}
155n/a
156n/a# Line terminators (we always output CRLF, but accept any of CRLF, CR, LF)
157n/a_CRLF = b'\r\n'
158n/a
159n/aGroupInfo = collections.namedtuple('GroupInfo',
160n/a ['group', 'last', 'first', 'flag'])
161n/a
162n/aArticleInfo = collections.namedtuple('ArticleInfo',
163n/a ['number', 'message_id', 'lines'])
164n/a
165n/a
166n/a# Helper function(s)
167n/adef decode_header(header_str):
168n/a """Takes a unicode string representing a munged header value
169n/a and decodes it as a (possibly non-ASCII) readable value."""
170n/a parts = []
171n/a for v, enc in _email_decode_header(header_str):
172n/a if isinstance(v, bytes):
173n/a parts.append(v.decode(enc or 'ascii'))
174n/a else:
175n/a parts.append(v)
176n/a return ''.join(parts)
177n/a
178n/adef _parse_overview_fmt(lines):
179n/a """Parse a list of string representing the response to LIST OVERVIEW.FMT
180n/a and return a list of header/metadata names.
181n/a Raises NNTPDataError if the response is not compliant
182n/a (cf. RFC 3977, section 8.4)."""
183n/a fmt = []
184n/a for line in lines:
185n/a if line[0] == ':':
186n/a # Metadata name (e.g. ":bytes")
187n/a name, _, suffix = line[1:].partition(':')
188n/a name = ':' + name
189n/a else:
190n/a # Header name (e.g. "Subject:" or "Xref:full")
191n/a name, _, suffix = line.partition(':')
192n/a name = name.lower()
193n/a name = _OVERVIEW_FMT_ALTERNATIVES.get(name, name)
194n/a # Should we do something with the suffix?
195n/a fmt.append(name)
196n/a defaults = _DEFAULT_OVERVIEW_FMT
197n/a if len(fmt) < len(defaults):
198n/a raise NNTPDataError("LIST OVERVIEW.FMT response too short")
199n/a if fmt[:len(defaults)] != defaults:
200n/a raise NNTPDataError("LIST OVERVIEW.FMT redefines default fields")
201n/a return fmt
202n/a
203n/adef _parse_overview(lines, fmt, data_process_func=None):
204n/a """Parse the response to an OVER or XOVER command according to the
205n/a overview format `fmt`."""
206n/a n_defaults = len(_DEFAULT_OVERVIEW_FMT)
207n/a overview = []
208n/a for line in lines:
209n/a fields = {}
210n/a article_number, *tokens = line.split('\t')
211n/a article_number = int(article_number)
212n/a for i, token in enumerate(tokens):
213n/a if i >= len(fmt):
214n/a # XXX should we raise an error? Some servers might not
215n/a # support LIST OVERVIEW.FMT and still return additional
216n/a # headers.
217n/a continue
218n/a field_name = fmt[i]
219n/a is_metadata = field_name.startswith(':')
220n/a if i >= n_defaults and not is_metadata:
221n/a # Non-default header names are included in full in the response
222n/a # (unless the field is totally empty)
223n/a h = field_name + ": "
224n/a if token and token[:len(h)].lower() != h:
225n/a raise NNTPDataError("OVER/XOVER response doesn't include "
226n/a "names of additional headers")
227n/a token = token[len(h):] if token else None
228n/a fields[fmt[i]] = token
229n/a overview.append((article_number, fields))
230n/a return overview
231n/a
232n/adef _parse_datetime(date_str, time_str=None):
233n/a """Parse a pair of (date, time) strings, and return a datetime object.
234n/a If only the date is given, it is assumed to be date and time
235n/a concatenated together (e.g. response to the DATE command).
236n/a """
237n/a if time_str is None:
238n/a time_str = date_str[-6:]
239n/a date_str = date_str[:-6]
240n/a hours = int(time_str[:2])
241n/a minutes = int(time_str[2:4])
242n/a seconds = int(time_str[4:])
243n/a year = int(date_str[:-4])
244n/a month = int(date_str[-4:-2])
245n/a day = int(date_str[-2:])
246n/a # RFC 3977 doesn't say how to interpret 2-char years. Assume that
247n/a # there are no dates before 1970 on Usenet.
248n/a if year < 70:
249n/a year += 2000
250n/a elif year < 100:
251n/a year += 1900
252n/a return datetime.datetime(year, month, day, hours, minutes, seconds)
253n/a
254n/adef _unparse_datetime(dt, legacy=False):
255n/a """Format a date or datetime object as a pair of (date, time) strings
256n/a in the format required by the NEWNEWS and NEWGROUPS commands. If a
257n/a date object is passed, the time is assumed to be midnight (00h00).
258n/a
259n/a The returned representation depends on the legacy flag:
260n/a * if legacy is False (the default):
261n/a date has the YYYYMMDD format and time the HHMMSS format
262n/a * if legacy is True:
263n/a date has the YYMMDD format and time the HHMMSS format.
264n/a RFC 3977 compliant servers should understand both formats; therefore,
265n/a legacy is only needed when talking to old servers.
266n/a """
267n/a if not isinstance(dt, datetime.datetime):
268n/a time_str = "000000"
269n/a else:
270n/a time_str = "{0.hour:02d}{0.minute:02d}{0.second:02d}".format(dt)
271n/a y = dt.year
272n/a if legacy:
273n/a y = y % 100
274n/a date_str = "{0:02d}{1.month:02d}{1.day:02d}".format(y, dt)
275n/a else:
276n/a date_str = "{0:04d}{1.month:02d}{1.day:02d}".format(y, dt)
277n/a return date_str, time_str
278n/a
279n/a
280n/aif _have_ssl:
281n/a
282n/a def _encrypt_on(sock, context, hostname):
283n/a """Wrap a socket in SSL/TLS. Arguments:
284n/a - sock: Socket to wrap
285n/a - context: SSL context to use for the encrypted connection
286n/a Returns:
287n/a - sock: New, encrypted socket.
288n/a """
289n/a # Generate a default SSL context if none was passed.
290n/a if context is None:
291n/a context = ssl._create_stdlib_context()
292n/a return context.wrap_socket(sock, server_hostname=hostname)
293n/a
294n/a
295n/a# The classes themselves
296n/aclass _NNTPBase:
297n/a # UTF-8 is the character set for all NNTP commands and responses: they
298n/a # are automatically encoded (when sending) and decoded (and receiving)
299n/a # by this class.
300n/a # However, some multi-line data blocks can contain arbitrary bytes (for
301n/a # example, latin-1 or utf-16 data in the body of a message). Commands
302n/a # taking (POST, IHAVE) or returning (HEAD, BODY, ARTICLE) raw message
303n/a # data will therefore only accept and produce bytes objects.
304n/a # Furthermore, since there could be non-compliant servers out there,
305n/a # we use 'surrogateescape' as the error handler for fault tolerance
306n/a # and easy round-tripping. This could be useful for some applications
307n/a # (e.g. NNTP gateways).
308n/a
309n/a encoding = 'utf-8'
310n/a errors = 'surrogateescape'
311n/a
312n/a def __init__(self, file, host,
313n/a readermode=None, timeout=_GLOBAL_DEFAULT_TIMEOUT):
314n/a """Initialize an instance. Arguments:
315n/a - file: file-like object (open for read/write in binary mode)
316n/a - host: hostname of the server
317n/a - readermode: if true, send 'mode reader' command after
318n/a connecting.
319n/a - timeout: timeout (in seconds) used for socket connections
320n/a
321n/a readermode is sometimes necessary if you are connecting to an
322n/a NNTP server on the local machine and intend to call
323n/a reader-specific commands, such as `group'. If you get
324n/a unexpected NNTPPermanentErrors, you might need to set
325n/a readermode.
326n/a """
327n/a self.host = host
328n/a self.file = file
329n/a self.debugging = 0
330n/a self.welcome = self._getresp()
331n/a
332n/a # Inquire about capabilities (RFC 3977).
333n/a self._caps = None
334n/a self.getcapabilities()
335n/a
336n/a # 'MODE READER' is sometimes necessary to enable 'reader' mode.
337n/a # However, the order in which 'MODE READER' and 'AUTHINFO' need to
338n/a # arrive differs between some NNTP servers. If _setreadermode() fails
339n/a # with an authorization failed error, it will set this to True;
340n/a # the login() routine will interpret that as a request to try again
341n/a # after performing its normal function.
342n/a # Enable only if we're not already in READER mode anyway.
343n/a self.readermode_afterauth = False
344n/a if readermode and 'READER' not in self._caps:
345n/a self._setreadermode()
346n/a if not self.readermode_afterauth:
347n/a # Capabilities might have changed after MODE READER
348n/a self._caps = None
349n/a self.getcapabilities()
350n/a
351n/a # RFC 4642 2.2.2: Both the client and the server MUST know if there is
352n/a # a TLS session active. A client MUST NOT attempt to start a TLS
353n/a # session if a TLS session is already active.
354n/a self.tls_on = False
355n/a
356n/a # Log in and encryption setup order is left to subclasses.
357n/a self.authenticated = False
358n/a
359n/a def __enter__(self):
360n/a return self
361n/a
362n/a def __exit__(self, *args):
363n/a is_connected = lambda: hasattr(self, "file")
364n/a if is_connected():
365n/a try:
366n/a self.quit()
367n/a except (OSError, EOFError):
368n/a pass
369n/a finally:
370n/a if is_connected():
371n/a self._close()
372n/a
373n/a def getwelcome(self):
374n/a """Get the welcome message from the server
375n/a (this is read and squirreled away by __init__()).
376n/a If the response code is 200, posting is allowed;
377n/a if it 201, posting is not allowed."""
378n/a
379n/a if self.debugging: print('*welcome*', repr(self.welcome))
380n/a return self.welcome
381n/a
382n/a def getcapabilities(self):
383n/a """Get the server capabilities, as read by __init__().
384n/a If the CAPABILITIES command is not supported, an empty dict is
385n/a returned."""
386n/a if self._caps is None:
387n/a self.nntp_version = 1
388n/a self.nntp_implementation = None
389n/a try:
390n/a resp, caps = self.capabilities()
391n/a except (NNTPPermanentError, NNTPTemporaryError):
392n/a # Server doesn't support capabilities
393n/a self._caps = {}
394n/a else:
395n/a self._caps = caps
396n/a if 'VERSION' in caps:
397n/a # The server can advertise several supported versions,
398n/a # choose the highest.
399n/a self.nntp_version = max(map(int, caps['VERSION']))
400n/a if 'IMPLEMENTATION' in caps:
401n/a self.nntp_implementation = ' '.join(caps['IMPLEMENTATION'])
402n/a return self._caps
403n/a
404n/a def set_debuglevel(self, level):
405n/a """Set the debugging level. Argument 'level' means:
406n/a 0: no debugging output (default)
407n/a 1: print commands and responses but not body text etc.
408n/a 2: also print raw lines read and sent before stripping CR/LF"""
409n/a
410n/a self.debugging = level
411n/a debug = set_debuglevel
412n/a
413n/a def _putline(self, line):
414n/a """Internal: send one line to the server, appending CRLF.
415n/a The `line` must be a bytes-like object."""
416n/a line = line + _CRLF
417n/a if self.debugging > 1: print('*put*', repr(line))
418n/a self.file.write(line)
419n/a self.file.flush()
420n/a
421n/a def _putcmd(self, line):
422n/a """Internal: send one command to the server (through _putline()).
423n/a The `line` must be a unicode string."""
424n/a if self.debugging: print('*cmd*', repr(line))
425n/a line = line.encode(self.encoding, self.errors)
426n/a self._putline(line)
427n/a
428n/a def _getline(self, strip_crlf=True):
429n/a """Internal: return one line from the server, stripping _CRLF.
430n/a Raise EOFError if the connection is closed.
431n/a Returns a bytes object."""
432n/a line = self.file.readline(_MAXLINE +1)
433n/a if len(line) > _MAXLINE:
434n/a raise NNTPDataError('line too long')
435n/a if self.debugging > 1:
436n/a print('*get*', repr(line))
437n/a if not line: raise EOFError
438n/a if strip_crlf:
439n/a if line[-2:] == _CRLF:
440n/a line = line[:-2]
441n/a elif line[-1:] in _CRLF:
442n/a line = line[:-1]
443n/a return line
444n/a
445n/a def _getresp(self):
446n/a """Internal: get a response from the server.
447n/a Raise various errors if the response indicates an error.
448n/a Returns a unicode string."""
449n/a resp = self._getline()
450n/a if self.debugging: print('*resp*', repr(resp))
451n/a resp = resp.decode(self.encoding, self.errors)
452n/a c = resp[:1]
453n/a if c == '4':
454n/a raise NNTPTemporaryError(resp)
455n/a if c == '5':
456n/a raise NNTPPermanentError(resp)
457n/a if c not in '123':
458n/a raise NNTPProtocolError(resp)
459n/a return resp
460n/a
461n/a def _getlongresp(self, file=None):
462n/a """Internal: get a response plus following text from the server.
463n/a Raise various errors if the response indicates an error.
464n/a
465n/a Returns a (response, lines) tuple where `response` is a unicode
466n/a string and `lines` is a list of bytes objects.
467n/a If `file` is a file-like object, it must be open in binary mode.
468n/a """
469n/a
470n/a openedFile = None
471n/a try:
472n/a # If a string was passed then open a file with that name
473n/a if isinstance(file, (str, bytes)):
474n/a openedFile = file = open(file, "wb")
475n/a
476n/a resp = self._getresp()
477n/a if resp[:3] not in _LONGRESP:
478n/a raise NNTPReplyError(resp)
479n/a
480n/a lines = []
481n/a if file is not None:
482n/a # XXX lines = None instead?
483n/a terminators = (b'.' + _CRLF, b'.\n')
484n/a while 1:
485n/a line = self._getline(False)
486n/a if line in terminators:
487n/a break
488n/a if line.startswith(b'..'):
489n/a line = line[1:]
490n/a file.write(line)
491n/a else:
492n/a terminator = b'.'
493n/a while 1:
494n/a line = self._getline()
495n/a if line == terminator:
496n/a break
497n/a if line.startswith(b'..'):
498n/a line = line[1:]
499n/a lines.append(line)
500n/a finally:
501n/a # If this method created the file, then it must close it
502n/a if openedFile:
503n/a openedFile.close()
504n/a
505n/a return resp, lines
506n/a
507n/a def _shortcmd(self, line):
508n/a """Internal: send a command and get the response.
509n/a Same return value as _getresp()."""
510n/a self._putcmd(line)
511n/a return self._getresp()
512n/a
513n/a def _longcmd(self, line, file=None):
514n/a """Internal: send a command and get the response plus following text.
515n/a Same return value as _getlongresp()."""
516n/a self._putcmd(line)
517n/a return self._getlongresp(file)
518n/a
519n/a def _longcmdstring(self, line, file=None):
520n/a """Internal: send a command and get the response plus following text.
521n/a Same as _longcmd() and _getlongresp(), except that the returned `lines`
522n/a are unicode strings rather than bytes objects.
523n/a """
524n/a self._putcmd(line)
525n/a resp, list = self._getlongresp(file)
526n/a return resp, [line.decode(self.encoding, self.errors)
527n/a for line in list]
528n/a
529n/a def _getoverviewfmt(self):
530n/a """Internal: get the overview format. Queries the server if not
531n/a already done, else returns the cached value."""
532n/a try:
533n/a return self._cachedoverviewfmt
534n/a except AttributeError:
535n/a pass
536n/a try:
537n/a resp, lines = self._longcmdstring("LIST OVERVIEW.FMT")
538n/a except NNTPPermanentError:
539n/a # Not supported by server?
540n/a fmt = _DEFAULT_OVERVIEW_FMT[:]
541n/a else:
542n/a fmt = _parse_overview_fmt(lines)
543n/a self._cachedoverviewfmt = fmt
544n/a return fmt
545n/a
546n/a def _grouplist(self, lines):
547n/a # Parse lines into "group last first flag"
548n/a return [GroupInfo(*line.split()) for line in lines]
549n/a
550n/a def capabilities(self):
551n/a """Process a CAPABILITIES command. Not supported by all servers.
552n/a Return:
553n/a - resp: server response if successful
554n/a - caps: a dictionary mapping capability names to lists of tokens
555n/a (for example {'VERSION': ['2'], 'OVER': [], LIST: ['ACTIVE', 'HEADERS'] })
556n/a """
557n/a caps = {}
558n/a resp, lines = self._longcmdstring("CAPABILITIES")
559n/a for line in lines:
560n/a name, *tokens = line.split()
561n/a caps[name] = tokens
562n/a return resp, caps
563n/a
564n/a def newgroups(self, date, *, file=None):
565n/a """Process a NEWGROUPS command. Arguments:
566n/a - date: a date or datetime object
567n/a Return:
568n/a - resp: server response if successful
569n/a - list: list of newsgroup names
570n/a """
571n/a if not isinstance(date, (datetime.date, datetime.date)):
572n/a raise TypeError(
573n/a "the date parameter must be a date or datetime object, "
574n/a "not '{:40}'".format(date.__class__.__name__))
575n/a date_str, time_str = _unparse_datetime(date, self.nntp_version < 2)
576n/a cmd = 'NEWGROUPS {0} {1}'.format(date_str, time_str)
577n/a resp, lines = self._longcmdstring(cmd, file)
578n/a return resp, self._grouplist(lines)
579n/a
580n/a def newnews(self, group, date, *, file=None):
581n/a """Process a NEWNEWS command. Arguments:
582n/a - group: group name or '*'
583n/a - date: a date or datetime object
584n/a Return:
585n/a - resp: server response if successful
586n/a - list: list of message ids
587n/a """
588n/a if not isinstance(date, (datetime.date, datetime.date)):
589n/a raise TypeError(
590n/a "the date parameter must be a date or datetime object, "
591n/a "not '{:40}'".format(date.__class__.__name__))
592n/a date_str, time_str = _unparse_datetime(date, self.nntp_version < 2)
593n/a cmd = 'NEWNEWS {0} {1} {2}'.format(group, date_str, time_str)
594n/a return self._longcmdstring(cmd, file)
595n/a
596n/a def list(self, group_pattern=None, *, file=None):
597n/a """Process a LIST or LIST ACTIVE command. Arguments:
598n/a - group_pattern: a pattern indicating which groups to query
599n/a - file: Filename string or file object to store the result in
600n/a Returns:
601n/a - resp: server response if successful
602n/a - list: list of (group, last, first, flag) (strings)
603n/a """
604n/a if group_pattern is not None:
605n/a command = 'LIST ACTIVE ' + group_pattern
606n/a else:
607n/a command = 'LIST'
608n/a resp, lines = self._longcmdstring(command, file)
609n/a return resp, self._grouplist(lines)
610n/a
611n/a def _getdescriptions(self, group_pattern, return_all):
612n/a line_pat = re.compile('^(?P<group>[^ \t]+)[ \t]+(.*)$')
613n/a # Try the more std (acc. to RFC2980) LIST NEWSGROUPS first
614n/a resp, lines = self._longcmdstring('LIST NEWSGROUPS ' + group_pattern)
615n/a if not resp.startswith('215'):
616n/a # Now the deprecated XGTITLE. This either raises an error
617n/a # or succeeds with the same output structure as LIST
618n/a # NEWSGROUPS.
619n/a resp, lines = self._longcmdstring('XGTITLE ' + group_pattern)
620n/a groups = {}
621n/a for raw_line in lines:
622n/a match = line_pat.search(raw_line.strip())
623n/a if match:
624n/a name, desc = match.group(1, 2)
625n/a if not return_all:
626n/a return desc
627n/a groups[name] = desc
628n/a if return_all:
629n/a return resp, groups
630n/a else:
631n/a # Nothing found
632n/a return ''
633n/a
634n/a def description(self, group):
635n/a """Get a description for a single group. If more than one
636n/a group matches ('group' is a pattern), return the first. If no
637n/a group matches, return an empty string.
638n/a
639n/a This elides the response code from the server, since it can
640n/a only be '215' or '285' (for xgtitle) anyway. If the response
641n/a code is needed, use the 'descriptions' method.
642n/a
643n/a NOTE: This neither checks for a wildcard in 'group' nor does
644n/a it check whether the group actually exists."""
645n/a return self._getdescriptions(group, False)
646n/a
647n/a def descriptions(self, group_pattern):
648n/a """Get descriptions for a range of groups."""
649n/a return self._getdescriptions(group_pattern, True)
650n/a
651n/a def group(self, name):
652n/a """Process a GROUP command. Argument:
653n/a - group: the group name
654n/a Returns:
655n/a - resp: server response if successful
656n/a - count: number of articles
657n/a - first: first article number
658n/a - last: last article number
659n/a - name: the group name
660n/a """
661n/a resp = self._shortcmd('GROUP ' + name)
662n/a if not resp.startswith('211'):
663n/a raise NNTPReplyError(resp)
664n/a words = resp.split()
665n/a count = first = last = 0
666n/a n = len(words)
667n/a if n > 1:
668n/a count = words[1]
669n/a if n > 2:
670n/a first = words[2]
671n/a if n > 3:
672n/a last = words[3]
673n/a if n > 4:
674n/a name = words[4].lower()
675n/a return resp, int(count), int(first), int(last), name
676n/a
677n/a def help(self, *, file=None):
678n/a """Process a HELP command. Argument:
679n/a - file: Filename string or file object to store the result in
680n/a Returns:
681n/a - resp: server response if successful
682n/a - list: list of strings returned by the server in response to the
683n/a HELP command
684n/a """
685n/a return self._longcmdstring('HELP', file)
686n/a
687n/a def _statparse(self, resp):
688n/a """Internal: parse the response line of a STAT, NEXT, LAST,
689n/a ARTICLE, HEAD or BODY command."""
690n/a if not resp.startswith('22'):
691n/a raise NNTPReplyError(resp)
692n/a words = resp.split()
693n/a art_num = int(words[1])
694n/a message_id = words[2]
695n/a return resp, art_num, message_id
696n/a
697n/a def _statcmd(self, line):
698n/a """Internal: process a STAT, NEXT or LAST command."""
699n/a resp = self._shortcmd(line)
700n/a return self._statparse(resp)
701n/a
702n/a def stat(self, message_spec=None):
703n/a """Process a STAT command. Argument:
704n/a - message_spec: article number or message id (if not specified,
705n/a the current article is selected)
706n/a Returns:
707n/a - resp: server response if successful
708n/a - art_num: the article number
709n/a - message_id: the message id
710n/a """
711n/a if message_spec:
712n/a return self._statcmd('STAT {0}'.format(message_spec))
713n/a else:
714n/a return self._statcmd('STAT')
715n/a
716n/a def next(self):
717n/a """Process a NEXT command. No arguments. Return as for STAT."""
718n/a return self._statcmd('NEXT')
719n/a
720n/a def last(self):
721n/a """Process a LAST command. No arguments. Return as for STAT."""
722n/a return self._statcmd('LAST')
723n/a
724n/a def _artcmd(self, line, file=None):
725n/a """Internal: process a HEAD, BODY or ARTICLE command."""
726n/a resp, lines = self._longcmd(line, file)
727n/a resp, art_num, message_id = self._statparse(resp)
728n/a return resp, ArticleInfo(art_num, message_id, lines)
729n/a
730n/a def head(self, message_spec=None, *, file=None):
731n/a """Process a HEAD command. Argument:
732n/a - message_spec: article number or message id
733n/a - file: filename string or file object to store the headers in
734n/a Returns:
735n/a - resp: server response if successful
736n/a - ArticleInfo: (article number, message id, list of header lines)
737n/a """
738n/a if message_spec is not None:
739n/a cmd = 'HEAD {0}'.format(message_spec)
740n/a else:
741n/a cmd = 'HEAD'
742n/a return self._artcmd(cmd, file)
743n/a
744n/a def body(self, message_spec=None, *, file=None):
745n/a """Process a BODY command. Argument:
746n/a - message_spec: article number or message id
747n/a - file: filename string or file object to store the body in
748n/a Returns:
749n/a - resp: server response if successful
750n/a - ArticleInfo: (article number, message id, list of body lines)
751n/a """
752n/a if message_spec is not None:
753n/a cmd = 'BODY {0}'.format(message_spec)
754n/a else:
755n/a cmd = 'BODY'
756n/a return self._artcmd(cmd, file)
757n/a
758n/a def article(self, message_spec=None, *, file=None):
759n/a """Process an ARTICLE command. Argument:
760n/a - message_spec: article number or message id
761n/a - file: filename string or file object to store the article in
762n/a Returns:
763n/a - resp: server response if successful
764n/a - ArticleInfo: (article number, message id, list of article lines)
765n/a """
766n/a if message_spec is not None:
767n/a cmd = 'ARTICLE {0}'.format(message_spec)
768n/a else:
769n/a cmd = 'ARTICLE'
770n/a return self._artcmd(cmd, file)
771n/a
772n/a def slave(self):
773n/a """Process a SLAVE command. Returns:
774n/a - resp: server response if successful
775n/a """
776n/a return self._shortcmd('SLAVE')
777n/a
778n/a def xhdr(self, hdr, str, *, file=None):
779n/a """Process an XHDR command (optional server extension). Arguments:
780n/a - hdr: the header type (e.g. 'subject')
781n/a - str: an article nr, a message id, or a range nr1-nr2
782n/a - file: Filename string or file object to store the result in
783n/a Returns:
784n/a - resp: server response if successful
785n/a - list: list of (nr, value) strings
786n/a """
787n/a pat = re.compile('^([0-9]+) ?(.*)\n?')
788n/a resp, lines = self._longcmdstring('XHDR {0} {1}'.format(hdr, str), file)
789n/a def remove_number(line):
790n/a m = pat.match(line)
791n/a return m.group(1, 2) if m else line
792n/a return resp, [remove_number(line) for line in lines]
793n/a
794n/a def xover(self, start, end, *, file=None):
795n/a """Process an XOVER command (optional server extension) Arguments:
796n/a - start: start of range
797n/a - end: end of range
798n/a - file: Filename string or file object to store the result in
799n/a Returns:
800n/a - resp: server response if successful
801n/a - list: list of dicts containing the response fields
802n/a """
803n/a resp, lines = self._longcmdstring('XOVER {0}-{1}'.format(start, end),
804n/a file)
805n/a fmt = self._getoverviewfmt()
806n/a return resp, _parse_overview(lines, fmt)
807n/a
808n/a def over(self, message_spec, *, file=None):
809n/a """Process an OVER command. If the command isn't supported, fall
810n/a back to XOVER. Arguments:
811n/a - message_spec:
812n/a - either a message id, indicating the article to fetch
813n/a information about
814n/a - or a (start, end) tuple, indicating a range of article numbers;
815n/a if end is None, information up to the newest message will be
816n/a retrieved
817n/a - or None, indicating the current article number must be used
818n/a - file: Filename string or file object to store the result in
819n/a Returns:
820n/a - resp: server response if successful
821n/a - list: list of dicts containing the response fields
822n/a
823n/a NOTE: the "message id" form isn't supported by XOVER
824n/a """
825n/a cmd = 'OVER' if 'OVER' in self._caps else 'XOVER'
826n/a if isinstance(message_spec, (tuple, list)):
827n/a start, end = message_spec
828n/a cmd += ' {0}-{1}'.format(start, end or '')
829n/a elif message_spec is not None:
830n/a cmd = cmd + ' ' + message_spec
831n/a resp, lines = self._longcmdstring(cmd, file)
832n/a fmt = self._getoverviewfmt()
833n/a return resp, _parse_overview(lines, fmt)
834n/a
835n/a def xgtitle(self, group, *, file=None):
836n/a """Process an XGTITLE command (optional server extension) Arguments:
837n/a - group: group name wildcard (i.e. news.*)
838n/a Returns:
839n/a - resp: server response if successful
840n/a - list: list of (name,title) strings"""
841n/a warnings.warn("The XGTITLE extension is not actively used, "
842n/a "use descriptions() instead",
843n/a DeprecationWarning, 2)
844n/a line_pat = re.compile('^([^ \t]+)[ \t]+(.*)$')
845n/a resp, raw_lines = self._longcmdstring('XGTITLE ' + group, file)
846n/a lines = []
847n/a for raw_line in raw_lines:
848n/a match = line_pat.search(raw_line.strip())
849n/a if match:
850n/a lines.append(match.group(1, 2))
851n/a return resp, lines
852n/a
853n/a def xpath(self, id):
854n/a """Process an XPATH command (optional server extension) Arguments:
855n/a - id: Message id of article
856n/a Returns:
857n/a resp: server response if successful
858n/a path: directory path to article
859n/a """
860n/a warnings.warn("The XPATH extension is not actively used",
861n/a DeprecationWarning, 2)
862n/a
863n/a resp = self._shortcmd('XPATH {0}'.format(id))
864n/a if not resp.startswith('223'):
865n/a raise NNTPReplyError(resp)
866n/a try:
867n/a [resp_num, path] = resp.split()
868n/a except ValueError:
869n/a raise NNTPReplyError(resp)
870n/a else:
871n/a return resp, path
872n/a
873n/a def date(self):
874n/a """Process the DATE command.
875n/a Returns:
876n/a - resp: server response if successful
877n/a - date: datetime object
878n/a """
879n/a resp = self._shortcmd("DATE")
880n/a if not resp.startswith('111'):
881n/a raise NNTPReplyError(resp)
882n/a elem = resp.split()
883n/a if len(elem) != 2:
884n/a raise NNTPDataError(resp)
885n/a date = elem[1]
886n/a if len(date) != 14:
887n/a raise NNTPDataError(resp)
888n/a return resp, _parse_datetime(date, None)
889n/a
890n/a def _post(self, command, f):
891n/a resp = self._shortcmd(command)
892n/a # Raises a specific exception if posting is not allowed
893n/a if not resp.startswith('3'):
894n/a raise NNTPReplyError(resp)
895n/a if isinstance(f, (bytes, bytearray)):
896n/a f = f.splitlines()
897n/a # We don't use _putline() because:
898n/a # - we don't want additional CRLF if the file or iterable is already
899n/a # in the right format
900n/a # - we don't want a spurious flush() after each line is written
901n/a for line in f:
902n/a if not line.endswith(_CRLF):
903n/a line = line.rstrip(b"\r\n") + _CRLF
904n/a if line.startswith(b'.'):
905n/a line = b'.' + line
906n/a self.file.write(line)
907n/a self.file.write(b".\r\n")
908n/a self.file.flush()
909n/a return self._getresp()
910n/a
911n/a def post(self, data):
912n/a """Process a POST command. Arguments:
913n/a - data: bytes object, iterable or file containing the article
914n/a Returns:
915n/a - resp: server response if successful"""
916n/a return self._post('POST', data)
917n/a
918n/a def ihave(self, message_id, data):
919n/a """Process an IHAVE command. Arguments:
920n/a - message_id: message-id of the article
921n/a - data: file containing the article
922n/a Returns:
923n/a - resp: server response if successful
924n/a Note that if the server refuses the article an exception is raised."""
925n/a return self._post('IHAVE {0}'.format(message_id), data)
926n/a
927n/a def _close(self):
928n/a self.file.close()
929n/a del self.file
930n/a
931n/a def quit(self):
932n/a """Process a QUIT command and close the socket. Returns:
933n/a - resp: server response if successful"""
934n/a try:
935n/a resp = self._shortcmd('QUIT')
936n/a finally:
937n/a self._close()
938n/a return resp
939n/a
940n/a def login(self, user=None, password=None, usenetrc=True):
941n/a if self.authenticated:
942n/a raise ValueError("Already logged in.")
943n/a if not user and not usenetrc:
944n/a raise ValueError(
945n/a "At least one of `user` and `usenetrc` must be specified")
946n/a # If no login/password was specified but netrc was requested,
947n/a # try to get them from ~/.netrc
948n/a # Presume that if .netrc has an entry, NNRP authentication is required.
949n/a try:
950n/a if usenetrc and not user:
951n/a import netrc
952n/a credentials = netrc.netrc()
953n/a auth = credentials.authenticators(self.host)
954n/a if auth:
955n/a user = auth[0]
956n/a password = auth[2]
957n/a except OSError:
958n/a pass
959n/a # Perform NNTP authentication if needed.
960n/a if not user:
961n/a return
962n/a resp = self._shortcmd('authinfo user ' + user)
963n/a if resp.startswith('381'):
964n/a if not password:
965n/a raise NNTPReplyError(resp)
966n/a else:
967n/a resp = self._shortcmd('authinfo pass ' + password)
968n/a if not resp.startswith('281'):
969n/a raise NNTPPermanentError(resp)
970n/a # Capabilities might have changed after login
971n/a self._caps = None
972n/a self.getcapabilities()
973n/a # Attempt to send mode reader if it was requested after login.
974n/a # Only do so if we're not in reader mode already.
975n/a if self.readermode_afterauth and 'READER' not in self._caps:
976n/a self._setreadermode()
977n/a # Capabilities might have changed after MODE READER
978n/a self._caps = None
979n/a self.getcapabilities()
980n/a
981n/a def _setreadermode(self):
982n/a try:
983n/a self.welcome = self._shortcmd('mode reader')
984n/a except NNTPPermanentError:
985n/a # Error 5xx, probably 'not implemented'
986n/a pass
987n/a except NNTPTemporaryError as e:
988n/a if e.response.startswith('480'):
989n/a # Need authorization before 'mode reader'
990n/a self.readermode_afterauth = True
991n/a else:
992n/a raise
993n/a
994n/a if _have_ssl:
995n/a def starttls(self, context=None):
996n/a """Process a STARTTLS command. Arguments:
997n/a - context: SSL context to use for the encrypted connection
998n/a """
999n/a # Per RFC 4642, STARTTLS MUST NOT be sent after authentication or if
1000n/a # a TLS session already exists.
1001n/a if self.tls_on:
1002n/a raise ValueError("TLS is already enabled.")
1003n/a if self.authenticated:
1004n/a raise ValueError("TLS cannot be started after authentication.")
1005n/a resp = self._shortcmd('STARTTLS')
1006n/a if resp.startswith('382'):
1007n/a self.file.close()
1008n/a self.sock = _encrypt_on(self.sock, context, self.host)
1009n/a self.file = self.sock.makefile("rwb")
1010n/a self.tls_on = True
1011n/a # Capabilities may change after TLS starts up, so ask for them
1012n/a # again.
1013n/a self._caps = None
1014n/a self.getcapabilities()
1015n/a else:
1016n/a raise NNTPError("TLS failed to start.")
1017n/a
1018n/a
1019n/aclass NNTP(_NNTPBase):
1020n/a
1021n/a def __init__(self, host, port=NNTP_PORT, user=None, password=None,
1022n/a readermode=None, usenetrc=False,
1023n/a timeout=_GLOBAL_DEFAULT_TIMEOUT):
1024n/a """Initialize an instance. Arguments:
1025n/a - host: hostname to connect to
1026n/a - port: port to connect to (default the standard NNTP port)
1027n/a - user: username to authenticate with
1028n/a - password: password to use with username
1029n/a - readermode: if true, send 'mode reader' command after
1030n/a connecting.
1031n/a - usenetrc: allow loading username and password from ~/.netrc file
1032n/a if not specified explicitly
1033n/a - timeout: timeout (in seconds) used for socket connections
1034n/a
1035n/a readermode is sometimes necessary if you are connecting to an
1036n/a NNTP server on the local machine and intend to call
1037n/a reader-specific commands, such as `group'. If you get
1038n/a unexpected NNTPPermanentErrors, you might need to set
1039n/a readermode.
1040n/a """
1041n/a self.host = host
1042n/a self.port = port
1043n/a self.sock = socket.create_connection((host, port), timeout)
1044n/a file = None
1045n/a try:
1046n/a file = self.sock.makefile("rwb")
1047n/a _NNTPBase.__init__(self, file, host,
1048n/a readermode, timeout)
1049n/a if user or usenetrc:
1050n/a self.login(user, password, usenetrc)
1051n/a except:
1052n/a if file:
1053n/a file.close()
1054n/a self.sock.close()
1055n/a raise
1056n/a
1057n/a def _close(self):
1058n/a try:
1059n/a _NNTPBase._close(self)
1060n/a finally:
1061n/a self.sock.close()
1062n/a
1063n/a
1064n/aif _have_ssl:
1065n/a class NNTP_SSL(_NNTPBase):
1066n/a
1067n/a def __init__(self, host, port=NNTP_SSL_PORT,
1068n/a user=None, password=None, ssl_context=None,
1069n/a readermode=None, usenetrc=False,
1070n/a timeout=_GLOBAL_DEFAULT_TIMEOUT):
1071n/a """This works identically to NNTP.__init__, except for the change
1072n/a in default port and the `ssl_context` argument for SSL connections.
1073n/a """
1074n/a self.sock = socket.create_connection((host, port), timeout)
1075n/a file = None
1076n/a try:
1077n/a self.sock = _encrypt_on(self.sock, ssl_context, host)
1078n/a file = self.sock.makefile("rwb")
1079n/a _NNTPBase.__init__(self, file, host,
1080n/a readermode=readermode, timeout=timeout)
1081n/a if user or usenetrc:
1082n/a self.login(user, password, usenetrc)
1083n/a except:
1084n/a if file:
1085n/a file.close()
1086n/a self.sock.close()
1087n/a raise
1088n/a
1089n/a def _close(self):
1090n/a try:
1091n/a _NNTPBase._close(self)
1092n/a finally:
1093n/a self.sock.close()
1094n/a
1095n/a __all__.append("NNTP_SSL")
1096n/a
1097n/a
1098n/a# Test retrieval when run as a script.
1099n/aif __name__ == '__main__':
1100n/a import argparse
1101n/a
1102n/a parser = argparse.ArgumentParser(description="""\
1103n/a nntplib built-in demo - display the latest articles in a newsgroup""")
1104n/a parser.add_argument('-g', '--group', default='gmane.comp.python.general',
1105n/a help='group to fetch messages from (default: %(default)s)')
1106n/a parser.add_argument('-s', '--server', default='news.gmane.org',
1107n/a help='NNTP server hostname (default: %(default)s)')
1108n/a parser.add_argument('-p', '--port', default=-1, type=int,
1109n/a help='NNTP port number (default: %s / %s)' % (NNTP_PORT, NNTP_SSL_PORT))
1110n/a parser.add_argument('-n', '--nb-articles', default=10, type=int,
1111n/a help='number of articles to fetch (default: %(default)s)')
1112n/a parser.add_argument('-S', '--ssl', action='store_true', default=False,
1113n/a help='use NNTP over SSL')
1114n/a args = parser.parse_args()
1115n/a
1116n/a port = args.port
1117n/a if not args.ssl:
1118n/a if port == -1:
1119n/a port = NNTP_PORT
1120n/a s = NNTP(host=args.server, port=port)
1121n/a else:
1122n/a if port == -1:
1123n/a port = NNTP_SSL_PORT
1124n/a s = NNTP_SSL(host=args.server, port=port)
1125n/a
1126n/a caps = s.getcapabilities()
1127n/a if 'STARTTLS' in caps:
1128n/a s.starttls()
1129n/a resp, count, first, last, name = s.group(args.group)
1130n/a print('Group', name, 'has', count, 'articles, range', first, 'to', last)
1131n/a
1132n/a def cut(s, lim):
1133n/a if len(s) > lim:
1134n/a s = s[:lim - 4] + "..."
1135n/a return s
1136n/a
1137n/a first = str(int(last) - args.nb_articles + 1)
1138n/a resp, overviews = s.xover(first, last)
1139n/a for artnum, over in overviews:
1140n/a author = decode_header(over['from']).split('<', 1)[0]
1141n/a subject = decode_header(over['subject'])
1142n/a lines = int(over[':lines'])
1143n/a print("{:7} {:20} {:42} ({})".format(
1144n/a artnum, cut(author, 20), cut(subject, 42), lines)
1145n/a )
1146n/a
1147n/a s.quit()