vf1 (45116B)
1 #!/usr/bin/env python3 2 # VF-1 Gopher client 3 # (C) 2018,2019 Solderpunk <solderpunk@sdf.org> 4 # With contributions from: 5 # - Alex Schroeder <alex@gnu.org> 6 # - Joseph Lyman <tfurrows@sdf.org> 7 # - Adam Mayer (https://github.com/phooky) 8 # - Paco Esteban <paco@onna.be> 9 10 import argparse 11 import cmd 12 import codecs 13 import collections 14 import fnmatch 15 import io 16 import mimetypes 17 import os.path 18 import random 19 import shlex 20 import shutil 21 import socket 22 import subprocess 23 import sys 24 import tempfile 25 import urllib.parse 26 import ssl 27 import time 28 29 _VERSION = "0.0.11" 30 31 # Use chardet if it's there, but don't depend on it 32 try: 33 import chardet 34 _HAS_CHARDET = True 35 except ImportError: 36 _HAS_CHARDET = False 37 38 # Command abbreviations 39 _ABBREVS = { 40 "a": "add", 41 "b": "back", 42 "bb": "blackbox", 43 "bm": "bookmarks", 44 "book": "bookmarks", 45 "f": "fold", 46 "fo": "forward", 47 "g": "go", 48 "h": "history", 49 "hist": "history", 50 "l": "less", 51 "li": "links", 52 "m": "mark", 53 "n": "next", 54 "p": "previous", 55 "prev": "previous", 56 "q": "quit", 57 "r": "reload", 58 "s": "save", 59 "se": "search", 60 "/": "search", 61 "t": "tour", 62 "u": "up", 63 "v": "veronica", 64 } 65 66 # Programs to handle different item types 67 _ITEMTYPE_TO_MIME = { 68 "1": "text/plain", 69 "0": "text/plain", 70 "h": "text/html", 71 "g": "image/gif", 72 } 73 74 _MIME_HANDLERS = { 75 "application/pdf": "xpdf %s", 76 "audio/mpeg": "mpg123 %s", 77 "audio/ogg": "ogg123 %s", 78 "image/*": "feh %s", 79 "text/*": "less --quit-if-one-screen %s", 80 "text/html": "lynx -dump -force_html %s", 81 } 82 83 # Item type formatting stuff 84 _ITEMTYPE_TITLES = { 85 "7": " <INP>", 86 "8": " <TEL>", 87 "9": " <BIN>", 88 "h": " <HTM>", 89 "g": " <IMG>", 90 "s": " <SND>", 91 "I": " <IMG>", 92 "T": " <TEL>", 93 } 94 95 _ANSI_COLORS = { 96 "red": "\x1b[0;31m", 97 "green": "\x1b[0;32m", 98 "yellow": "\x1b[0;33m", 99 "blue": "\x1b[0;34m", 100 "purple": "\x1b[0;35m", 101 "cyan": "\x1b[0;36m", 102 "white": "\x1b[0;37m", 103 "black": "\x1b[0;30m", 104 } 105 106 _ITEMTYPE_COLORS = { 107 "0": _ANSI_COLORS["green"], # Text File 108 "1": _ANSI_COLORS["blue"], # Sub-menu 109 "7": _ANSI_COLORS["red"], # Search / Input 110 "8": _ANSI_COLORS["purple"], # Telnet 111 "9": _ANSI_COLORS["cyan"], # Binary 112 "g": _ANSI_COLORS["blue"], # Gif 113 "h": _ANSI_COLORS["green"], # HTML 114 "s": _ANSI_COLORS["cyan"], # Sound 115 "I": _ANSI_COLORS["cyan"], # Gif 116 "T": _ANSI_COLORS["purple"], # Telnet 117 } 118 119 CRLF = '\r\n' 120 121 # Lightweight representation of an item in Gopherspace 122 GopherItem = collections.namedtuple("GopherItem", 123 ("host", "port", "path", "itemtype", "name")) 124 125 def url_to_gopheritem(url): 126 # urllibparse.urlparse can handle IPv6 addresses, but only if they 127 # are formatted very carefully, in a way that users almost 128 # certainly won't expect. So, catch them early and try to fix 129 # them... 130 if url.count(":") > 2: # Best way to detect them? 131 url = fix_ipv6_url(url) 132 # Prepend a gopher schema if none given 133 if "://" not in url: 134 url = "gopher://" + url 135 u = urllib.parse.urlparse(url) 136 # https://tools.ietf.org/html/rfc4266#section-2.1 137 path = u.path 138 if u.path and u.path[0] == '/' and len(u.path) > 1: 139 itemtype = u.path[1] 140 path = u.path[2:] 141 else: 142 # Use item type 1 for top-level selector 143 itemtype = 1 144 return GopherItem(u.hostname, u.port or 70, path, 145 str(itemtype), "") 146 147 def fix_ipv6_url(url): 148 # If there's a pair of []s in there, it's probably fine as is. 149 if "[" in url and "]" in url: 150 return url 151 # Easiest case is a raw address, no schema, no path. 152 # Just wrap it in square brackets and whack a slash on the end 153 if "/" not in url: 154 return "[" + url + "]/" 155 # Now the trickier cases... 156 if "://" in url: 157 schema, schemaless = url.split("://") 158 else: 159 schema, schemaless = None, url 160 if "/" in schemaless: 161 netloc, rest = schemaless.split("/",1) 162 schemaless = "[" + netloc + "]" + "/" + rest 163 if schema: 164 return schema + "://" + schemaless 165 return schemaless 166 167 def gopheritem_to_url(gi): 168 if gi and gi.host: 169 return ("gopher://%s:%d/%s%s" % ( 170 gi.host, int(gi.port), 171 gi.itemtype, gi.path)) 172 elif gi: 173 return gi.path 174 else: 175 return "" 176 177 def gopheritem_from_line(line): 178 # Split on tabs. Strip final element after splitting, 179 # since if we split first we loose empty elements. 180 parts = line.split("\t") 181 parts[-1] = parts[-1].strip() 182 # Discard Gopher+ noise 183 if parts[-1] == "+": 184 parts = parts[:-1] 185 # Attempt to assign variables. This may fail. 186 # It's up to the caller to catch the Exception. 187 name, path, host, port = parts 188 itemtype = name[0] 189 name = name[1:] 190 port = int(port) 191 # Handle the h-type URL: hack for secure links 192 if itemtype == "h" and path.startswith("URL:gopher"): 193 url = path[4:] 194 return url_to_gopheritem(url) 195 return GopherItem(host, port, path, itemtype, name) 196 197 def gopheritem_to_line(gi, name=""): 198 name = ((name or gi.name) or gopheritem_to_url(gi)) 199 # Prepend itemtype to name 200 name = str(gi.itemtype) + name 201 path = gi.path 202 return "\t".join((name, path, gi.host or "", str(gi.port))) + "\n" 203 204 # Cheap and cheerful URL detector 205 def looks_like_url(word): 206 return "." in word and ("gopher://" in word or "gophers://" in word) 207 208 def extract_url(word): 209 # Given a word that probably contains a URL, extract that URL from 210 # with sensible surrounding punctuation. 211 for start, end in (("<",">"), ('[',']'), ("(",")"), ("'","'"), ('"','"')): 212 if word[0] == start and end in word: 213 return word[1:word.rfind(end)] 214 if word.endswith("."): 215 return word[:-1] 216 else: 217 return word 218 219 # Decorators 220 def needs_gi(inner): 221 def outer(self, *args, **kwargs): 222 if not self.gi: 223 print("You need to 'go' somewhere, first") 224 return None 225 else: 226 return inner(self, *args, **kwargs) 227 outer.__doc__ = inner.__doc__ 228 return outer 229 230 class GopherClient(cmd.Cmd): 231 232 def __init__(self, debug=False, tls=False): 233 cmd.Cmd.__init__(self) 234 self._set_tls(tls) 235 self.gi = None 236 self.history = [] 237 self.hist_index = 0 238 self.menu_filename = "" 239 self.menu = [] 240 self.menu_index = -1 241 self.lookup = self.menu 242 self.marks = {} 243 self.mirrors = {} 244 self.page_index = 0 245 self.tmp_filename = "" 246 self.visited_hosts = set() 247 self.waypoints = [] 248 249 self.options = { 250 "color_menus" : False, 251 "debug" : debug, 252 "encoding" : "iso-8859-1", 253 "ipv6" : True, 254 "timeout" : 10, 255 } 256 257 self.log = { 258 "start_time": time.time(), 259 "requests": 0, 260 "tls_requests": 0, 261 "ipv4_requests": 0, 262 "ipv6_requests": 0, 263 "bytes_recvd": 0, 264 "ipv4_bytes_recvd": 0, 265 "ipv6_bytes_recvd": 0, 266 "dns_failures": 0, 267 "refused_connections": 0, 268 "reset_connections": 0, 269 "timeouts": 0, 270 } 271 self.itemtype_counts = { } 272 273 def _go_to_gi(self, gi, update_hist=True, query_str=None, handle=True): 274 """This method might be considered "the heart of VF-1". 275 Everything involved in fetching a gopher resource happens here: 276 sending the request over the network, parsing the response if 277 its a menu, storing the response in a temporary file, choosing 278 and calling a handler program, and updating the history.""" 279 # Handle non-gopher protocols first 280 if gi.itemtype in ("8", "T", "S"): 281 # SSH (non-standard, but nice) 282 if gi.itemtype == "S": 283 subprocess.call(shlex.split("ssh %s@%s -p %s" % (gi.path, gi.host, gi.port))) 284 # Telnet 285 elif gi.path: 286 subprocess.call(shlex.split("telnet -l %s %s %s" % (gi.path, gi.host, gi.port))) 287 else: 288 subprocess.call(shlex.split("telnet %s %s" % (gi.host, gi.port))) 289 if update_hist: 290 self._update_history(gi) 291 return 292 293 # From here on in, it's gopher only 294 295 # Do everything which touches the network in one block, 296 # so we only need to catch exceptions once 297 try: 298 # Is this a local file? 299 if not gi.host: 300 address, f = None, open(gi.path, "rb") 301 # Is this a search point? 302 elif gi.itemtype == "7": 303 if not query_str: 304 query_str = input("Query term: ") 305 address, f = self._send_request(gi, query=query_str) 306 else: 307 address, f = self._send_request(gi) 308 # Read whole response 309 response = f.read() 310 f.close() 311 312 # Catch network errors which may be recoverable if a redundant 313 # mirror is specified 314 except (socket.gaierror, ConnectionRefusedError, 315 ConnectionResetError, TimeoutError, socket.timeout, 316 ) as network_error: 317 # Print an error message 318 if isinstance(network_error, socket.gaierror): 319 self.log["dns_failures"] += 1 320 print("ERROR: DNS error!") 321 elif isinstance(network_error, ConnectionRefusedError): 322 self.log["refused_connections"] += 1 323 print("ERROR: Connection refused!") 324 elif isinstance(network_error, ConnectionResetError): 325 self.log["reset_connections"] += 1 326 print("ERROR: Connection reset!") 327 elif isinstance(network_error, (TimeoutError, socket.timeout)): 328 self.log["timeouts"] += 1 329 print("""ERROR: Connection timed out! 330 Slow internet connection? Use 'set timeout' to be more patient.""") 331 if not self.tls: 332 print("Encrypted gopher server? Use 'tls' to enable encryption.") 333 # Try to fall back on a redundant mirror 334 new_gi = self._get_mirror_gi(gi) 335 if new_gi: 336 print("Trying redundant mirror %s..." % gopheritem_to_url(new_gi)) 337 self._go_to_gi(new_gi) 338 return 339 # Catch non-recoverable errors 340 except Exception as err: 341 print("ERROR: " + str(err)) 342 if isinstance(err, ssl.SSLError): 343 print(gopheritem_to_url(gi) + " is probably not encrypted.") 344 print("Use 'tls' to disable encryption.") 345 return 346 347 # Attempt to decode something that is supposed to be text 348 if gi.itemtype in ("0", "1", "7", "h"): 349 try: 350 response = self._decode_text(response) 351 except UnicodeError: 352 print("""ERROR: Unknown text encoding! 353 If you know the correct encoding, use e.g. 'set encoding koi8-r' and 354 try again. Otherwise, install the 'chardet' library for Python 3 to 355 enable automatic encoding detection.""") 356 return 357 358 # Render gopher menus 359 if gi.itemtype in ("1", "7"): 360 response = self._render_menu(response, gi) 361 362 # Save the result in a temporary file 363 ## Delete old file 364 if self.tmp_filename: 365 os.unlink(self.tmp_filename) 366 ## Set file mode 367 if gi.itemtype in ("0", "1", "7", "h"): 368 mode = "w" 369 encoding = "UTF-8" 370 else: 371 mode = "wb" 372 encoding = None 373 ## Write 374 tmpf = tempfile.NamedTemporaryFile(mode, encoding=encoding, delete=False) 375 size = tmpf.write(response) 376 tmpf.close() 377 self.tmp_filename = tmpf.name 378 self._debug("Wrote %d byte response to %s." % (size, self.tmp_filename)) 379 380 # Pass file to handler, unless we were asked not to 381 if handle: 382 cmd_str = self._get_handler_cmd(gi) 383 try: 384 subprocess.call(cmd_str % tmpf.name, shell=True) 385 except FileNotFoundError: 386 print("Handler program %s not found!" % shlex.split(cmd_str)[0]) 387 print("You can use the ! command to specify another handler program or pipeline.") 388 389 # Update state 390 self.gi = gi 391 self._log_visit(gi, address, size) 392 if update_hist: 393 self._update_history(gi) 394 395 # This method below started life as the core of the old gopherlib.py 396 # module from Python 2.4, with minimal changes made for Python 3 397 # compatibility and to handle convenient download of plain text (including 398 # Unicode) or binary files. It's come a long way since then, though. 399 # Everything network related happens in this one method! 400 def _send_request(self, gi, query=None): 401 """Send a selector to a given host and port. 402 Returns the resolved address and binary file with the reply.""" 403 # Add query to selector 404 if query: 405 gi = gi._replace(path=gi.path + "\t" + query) 406 # DNS lookup - will get IPv4 and IPv6 records if IPv6 is enabled 407 if ":" in gi.host: 408 # This is likely a literal IPv6 address, so we can *only* ask for 409 # IPv6 addresses or getaddrinfo will complain 410 family_mask = socket.AF_INET6 411 elif socket.has_ipv6 and self.options["ipv6"]: 412 # Accept either IPv4 or IPv6 addresses 413 family_mask = 0 414 else: 415 # IPv4 only 416 family_mask = socket.AF_INET 417 addresses = socket.getaddrinfo(gi.host, gi.port, family=family_mask, 418 type=socket.SOCK_STREAM) 419 # Sort addresses so IPv6 ones come first 420 addresses.sort(key=lambda add: add[0] == socket.AF_INET6, reverse=True) 421 # Verify that this sort works 422 if any(add[0] == socket.AF_INET6 for add in addresses): 423 assert addresses[0][0] == socket.AF_INET6 424 # Connect to remote host by any address possible 425 err = None 426 for address in addresses: 427 self._debug("Connecting to: " + str(address[4])) 428 s = socket.socket(address[0], address[1]) 429 s.settimeout(self.options["timeout"]) 430 if self.tls: 431 context = ssl.create_default_context() 432 # context.check_hostname = False 433 # context.verify_mode = ssl.CERT_NONE 434 s = context.wrap_socket(s, server_hostname = gi.host) 435 try: 436 s.connect(address[4]) 437 break 438 except OSError as e: 439 err = e 440 else: 441 # If we couldn't connect to *any* of the addresses, just 442 # bubble up the exception from the last attempt and deny 443 # knowledge of earlier failures. 444 raise err 445 # Send request and wrap response in a file descriptor 446 self._debug("Sending %s<CRLF>" % gi.path) 447 s.sendall((gi.path + CRLF).encode("UTF-8")) 448 return address, s.makefile(mode = "rb") 449 450 def _get_handler_cmd(self, gi): 451 # First, get mimetype, either from itemtype or filename 452 if gi.itemtype in _ITEMTYPE_TO_MIME: 453 mimetype = _ITEMTYPE_TO_MIME[gi.itemtype] 454 else: 455 mimetype, encoding = mimetypes.guess_type(gi.path) 456 if mimetype is None: 457 # No idea what this is, try harder by looking at the 458 # magic number using file(1) 459 out = subprocess.check_output( 460 shlex.split("file --brief --mime-type %s" % self.tmp_filename)) 461 mimetype = out.decode("UTF-8").strip() 462 # Don't permit file extensions to completely override the 463 # vaguer imagetypes 464 if gi.itemtype == "I" and not mimetype.startswith("image"): 465 # The server declares this to be an image. 466 # But it has a weird or missing file extension, so the 467 # MIME type was guessed as something else. 468 # We shall trust the server that it's an image. 469 # Pretend it's a jpeg, because whatever handler the user has 470 # set for jpegs probably has the best shot at handling this. 471 mimetype = "image/jpeg" 472 elif gi.itemtype == "s" and not mimetype.startswith("audio"): 473 # As above, this is "weird audio". 474 # Pretend it's an mp3? 475 mimetype = "audio/mpeg" 476 self._debug("Assigned MIME type: %s" % mimetype) 477 478 # Now look for a handler for this mimetype 479 # Consider exact matches before wildcard matches 480 exact_matches = [] 481 wildcard_matches = [] 482 for handled_mime, cmd_str in _MIME_HANDLERS.items(): 483 if "*" in handled_mime: 484 wildcard_matches.append((handled_mime, cmd_str)) 485 else: 486 exact_matches.append((handled_mime, cmd_str)) 487 for handled_mime, cmd_str in exact_matches + wildcard_matches: 488 if fnmatch.fnmatch(mimetype, handled_mime): 489 break 490 else: 491 # Use "xdg-open" as a last resort. 492 cmd_str = "xdg-open %s" 493 self._debug("Using handler: %s" % cmd_str) 494 return cmd_str 495 496 def _decode_text(self, raw_bytes): 497 # Attempt to decode some bytes into a Unicode string. 498 # First of all, try UTF-8 as the default. 499 # If this fails, attempt to autodetect the encoding if chardet 500 # library is installed. 501 # If chardet is not installed, or fails to work, fall back on 502 # the user-specified alternate encoding. 503 # If none of this works, this will raise UnicodeError and it's 504 # up to the caller to handle it gracefully. 505 # Try UTF-8 first: 506 try: 507 text = raw_bytes.decode("UTF-8") 508 except UnicodeError: 509 # If we have chardet, try the magic 510 self._debug("Could not decode response as UTF-8.") 511 if _HAS_CHARDET: 512 autodetect = chardet.detect(raw_bytes) 513 # Make sure we're vaguely certain 514 if autodetect["confidence"] > 0.5: 515 self._debug("Trying encoding %s as recommended by chardet." % autodetect["encoding"]) 516 text = raw_bytes.decode(autodetect["encoding"]) 517 else: 518 # Try the user-specified encoding 519 self._debug("Trying fallback encoding %s." % self.options["encoding"]) 520 text = raw_bytes.decode(self.options["encoding"]) 521 else: 522 # Try the user-specified encoding 523 text = raw_bytes.decode(self.options["encoding"]) 524 if not text.endswith("\n"): 525 text += CRLF 526 return text 527 528 def _render_menu(self, response, menu_gi): 529 self.menu = [] 530 if self.menu_filename: 531 os.unlink(self.menu_filename) 532 tmpf = tempfile.NamedTemporaryFile("w", encoding="UTF-8", delete=False) 533 self.menu_filename = tmpf.name 534 response = io.StringIO(response) 535 rendered = [] 536 for line in response.readlines(): 537 if line.startswith("3"): 538 print("Error message from server:") 539 print(line[1:].split("\t")[0]) 540 tmpf.close() 541 os.unlink(self.menu_filename) 542 self.menu_filename = "" 543 return "" 544 else: 545 tmpf.write(line) 546 547 if line.startswith("i"): 548 rendered.append(line[1:].split("\t")[0] + "\n") 549 else: 550 try: 551 gi = gopheritem_from_line(line) 552 except: 553 # Silently ignore things which are not errors, information 554 # lines or things which look like valid menu items 555 self._debug("Ignoring menu line: %s" % line) 556 continue 557 if gi.itemtype == "+": 558 self._register_redundant_server(gi) 559 continue 560 self.menu.append(gi) 561 rendered.append(self._format_gopheritem(len(self.menu), gi) + "\n") 562 563 self.lookup = self.menu 564 self.page_index = 0 565 self.menu_index = -1 566 567 return "".join(rendered) 568 569 def _format_gopheritem(self, index, gi, url=False): 570 line = "[%d] " % index 571 # Add item name, with itemtype indicator for non-text items 572 if gi.name: 573 line += gi.name 574 # Use URL in place of name if we didn't get here from a menu 575 else: 576 line += gopheritem_to_url(gi) 577 if gi.itemtype in _ITEMTYPE_TITLES: 578 line += _ITEMTYPE_TITLES[gi.itemtype] 579 elif gi.itemtype == "1" and not line.endswith("/"): 580 line += "/" 581 # Add URL if requested 582 if gi.name and url: 583 line += " (%s)" % gopheritem_to_url(gi) 584 # Colourise 585 if self.options["color_menus"] and gi.itemtype in _ITEMTYPE_COLORS: 586 line = _ITEMTYPE_COLORS[gi.itemtype] + line + "\x1b[0m" 587 return line 588 589 def _register_redundant_server(self, gi): 590 # This mirrors the last non-mirror item 591 target = self.menu[-1] 592 target = (target.host, target.port, target.path) 593 if target not in self.mirrors: 594 self.mirrors[target] = [] 595 self.mirrors[target].append((gi.host, gi.port, gi.path)) 596 self._debug("Registered redundant mirror %s" % gopheritemi_to_url(gi)) 597 598 def _get_mirror_gi(self, gi): 599 # Search for a redundant mirror that matches this GI 600 for (host, port, path_prefix), mirrors in self.mirrors.items(): 601 if (host == gi.host and port == gi.port and 602 gi.path.startswith(path_prefix)): 603 break 604 else: 605 # If there are no mirrors, we're done 606 return None 607 # Pick a mirror at random and build a new GI for it 608 mirror_host, mirror_port, mirror_path = random.sample(mirrors, 1)[0] 609 new_gi = GopherItem(mirror_host, mirror_port, 610 mirror_path + "/" + gi.path[len(path_prefix):], 611 gi.itemtype, gi.name) 612 return new_gi 613 614 def _show_lookup(self, offset=0, end=None, url=False): 615 for n, gi in enumerate(self.lookup[offset:end]): 616 print(self._format_gopheritem(n+offset+1, gi, url)) 617 618 def _update_history(self, gi): 619 # Don't duplicate 620 if self.history and self.history[self.hist_index] == gi: 621 return 622 self.history = self.history[0:self.hist_index+1] 623 self.history.append(gi) 624 self.hist_index = len(self.history) - 1 625 626 def _log_visit(self, gi, address, size): 627 self.itemtype_counts[gi.itemtype] = ( 628 self.itemtype_counts.get(gi.itemtype, 0) + 1) 629 if not address: 630 return 631 self.log["requests"] += 1 632 self.log["bytes_recvd"] += size 633 self.visited_hosts.add(address) 634 if self.tls: 635 self.log["tls_requests"] += 1 636 if address[0] == socket.AF_INET: 637 self.log["ipv4_requests"] += 1 638 self.log["ipv4_bytes_recvd"] += size 639 elif address[0] == socket.AF_INET6: 640 self.log["ipv6_requests"] += 1 641 self.log["ipv6_bytes_recvd"] += size 642 643 def _set_tls(self, tls): 644 self.tls = tls 645 if self.tls: 646 self.prompt = "\x1b[38;5;196m" + "VF-1" + "\x1b[38;5;255m" + "> " + "\x1b[0m" 647 else: 648 self.prompt = "\x1b[38;5;202m" + "VF-1" + "\x1b[38;5;255m" + "> " + "\x1b[0m" 649 650 def _debug(self, debug_text): 651 if not self.options["debug"]: 652 return 653 debug_text = "\x1b[0;32m[DEBUG] " + debug_text + "\x1b[0m" 654 print(debug_text) 655 656 # Cmd implementation follows 657 658 def default(self, line): 659 if line.strip() == "EOF": 660 return self.onecmd("quit") 661 elif line.strip() == "..": 662 return self.do_up() 663 elif line.startswith("/"): 664 return self.do_search(line[1:]) 665 666 # Expand abbreviated commands 667 first_word = line.split()[0].strip() 668 if first_word in _ABBREVS: 669 full_cmd = _ABBREVS[first_word] 670 expanded = line.replace(first_word, full_cmd, 1) 671 return self.onecmd(expanded) 672 673 # Try to parse numerical index for lookup table 674 try: 675 n = int(line.strip()) 676 except ValueError: 677 print("What?") 678 return 679 680 try: 681 gi = self.lookup[n-1] 682 except IndexError: 683 print ("Index too high!") 684 return 685 686 self.menu_index = n 687 self._go_to_gi(gi) 688 689 ### Settings 690 def do_set(self, line): 691 """View or set various options.""" 692 if not line.strip(): 693 # Show all current settings 694 for option in sorted(self.options.keys()): 695 print("%s %s" % (option, self.options[option])) 696 elif len(line.split()) == 1: 697 option = line.strip() 698 if option in self.options: 699 print("%s %s" % (option, self.options[option])) 700 else: 701 print("Unrecognised option %s" % option) 702 else: 703 option, value = line.split(" ", 1) 704 if option not in self.options: 705 print("Unrecognised option %s" % option) 706 return 707 elif option == "encoding": 708 try: 709 codecs.lookup(value) 710 except LookupError: 711 print("Unknown encoding %s" % value) 712 return 713 elif value.isnumeric(): 714 value = int(value) 715 elif value.lower() == "false": 716 value = False 717 elif value.lower() == "true": 718 value = True 719 else: 720 try: 721 value = float(value) 722 except ValueError: 723 pass 724 self.options[option] = value 725 726 def do_handler(self, line): 727 """View or set handler commands for different MIME types.""" 728 if not line.strip(): 729 # Show all current handlers 730 for mime in sorted(_MIME_HANDLERS.keys()): 731 print("%s %s" % (mime, _MIME_HANDLERS[mime])) 732 elif len(line.split()) == 1: 733 mime = line.strip() 734 if mime in _MIME_HANDLERS: 735 print("%s %s" % (mime, _MIME_HANDLERS[mime])) 736 else: 737 print("No handler set for MIME type %s" % mime) 738 else: 739 mime, handler = line.split(" ", 1) 740 _MIME_HANDLERS[mime] = handler 741 if "%s" not in handler: 742 print("Are you sure you don't want to pass the filename to the handler?") 743 744 ### Stuff for getting around 745 def do_go(self, line): 746 """Go to a gopher URL or marked item.""" 747 line = line.strip() 748 if not line: 749 print("Go where?") 750 # First, check for possible marks 751 elif line in self.marks: 752 gi = self.marks[line] 753 self._go_to_gi(gi) 754 # or a local file 755 elif os.path.exists(os.path.expanduser(line)): 756 gi = GopherItem(None, None, os.path.expanduser(line), 757 "1", line) 758 self._go_to_gi(gi) 759 # If this isn't a mark, treat it as a URL 760 else: 761 url = line 762 gi = url_to_gopheritem(url) 763 self._go_to_gi(gi) 764 765 @needs_gi 766 def do_reload(self, *args): 767 """Reload the current URL.""" 768 self._go_to_gi(self.gi) 769 770 @needs_gi 771 def do_up(self, *args): 772 """Go up one directory in the path.""" 773 new_path, removed = os.path.split(self.gi.path) 774 if not removed: 775 new_path, removed = os.path.split(new_path) 776 new_gi = self.gi._replace(path=new_path, itemtype="1") 777 self._go_to_gi(new_gi) 778 779 def do_back(self, *args): 780 """Go back to the previous gopher item.""" 781 if not self.history or self.hist_index == 0: 782 return 783 self.hist_index -= 1 784 gi = self.history[self.hist_index] 785 self._go_to_gi(gi, update_hist=False) 786 787 def do_forward(self, *args): 788 """Go forward to the next gopher item.""" 789 if not self.history or self.hist_index == len(self.history) - 1: 790 return 791 self.hist_index += 1 792 gi = self.history[self.hist_index] 793 self._go_to_gi(gi, update_hist=False) 794 795 def do_next(self, *args): 796 """Go to next item after current in index.""" 797 return self.onecmd(str(self.menu_index+1)) 798 799 def do_previous(self, *args): 800 """Go to previous item before current in index.""" 801 self.lookup = self.menu 802 return self.onecmd(str(self.menu_index-1)) 803 804 @needs_gi 805 def do_root(self, *args): 806 """Go to root selector of the server hosting current item.""" 807 gi = self.gi._replace(path="", itemtype="1", name=self.gi.host) 808 self._go_to_gi(gi) 809 810 def do_tour(self, line): 811 """Add index items as waypoints on a tour, which is basically a FIFO 812 queue of gopher items. 813 814 Items can be added with `tour 1 2 3 4` or ranges like `tour 1-4`. 815 All items in current menu can be added with `tour *`. 816 Current tour can be listed with `tour ls` and scrubbed with `tour clear`.""" 817 line = line.strip() 818 if not line: 819 # Fly to next waypoint on tour 820 if not self.waypoints: 821 print("End of tour.") 822 else: 823 gi = self.waypoints.pop(0) 824 self._go_to_gi(gi) 825 elif line == "ls": 826 old_lookup = self.lookup 827 self.lookup = self.waypoints 828 self._show_lookup() 829 self.lookup = old_lookup 830 elif line == "clear": 831 self.waypoints = [] 832 elif line == "*": 833 self.waypoints.extend(self.lookup) 834 elif looks_like_url(line): 835 self.waypoints.append(url_to_gopheritem(line)) 836 else: 837 for index in line.split(): 838 try: 839 pair = index.split('-') 840 if len(pair) == 1: 841 # Just a single index 842 n = int(index) 843 gi = self.lookup[n-1] 844 self.waypoints.append(gi) 845 elif len(pair) == 2: 846 # Two endpoints for a range of indices 847 for n in range(int(pair[0]), int(pair[1]) + 1): 848 gi = self.lookup[n-1] 849 self.waypoints.append(gi) 850 else: 851 # Syntax error 852 print("Invalid use of range syntax %s, skipping" % index) 853 except ValueError: 854 print("Non-numeric index %s, skipping." % index) 855 except IndexError: 856 print("Invalid index %d, skipping." % n) 857 858 @needs_gi 859 def do_mark(self, line): 860 """Mark the current item with a single letter. This letter can then 861 be passed to the 'go' command to return to the current item later. 862 Think of it like marks in vi: 'mark a'='ma' and 'go a'=''a'.""" 863 line = line.strip() 864 if not line: 865 for mark, gi in self.marks.items(): 866 print("[%s] %s (%s)" % (mark, gi.name, gopheritem_to_url(gi))) 867 elif line.isalpha() and len(line) == 1: 868 self.marks[line] = self.gi 869 else: 870 print("Invalid mark, must be one letter") 871 872 def do_veronica(self, line): 873 # Don't tell Betty! 874 """Submit a search query to the Veronica 2 search engine.""" 875 veronica = url_to_gopheritem("gopher://gopher.floodgap.com:70/7/v2/vs") 876 self._go_to_gi(veronica, query_str = line) 877 878 def do_version(self, line): 879 """Display version information.""" 880 print("VF-1 " + _VERSION) 881 882 ### Stuff that modifies the lookup table 883 def do_ls(self, line): 884 """List contents of current index. 885 Use 'ls -l' to see URLs.""" 886 self.lookup = self.menu 887 self._show_lookup(url = "-l" in line) 888 self.page_index = 0 889 890 def do_history(self, *args): 891 """Display history.""" 892 self.lookup = self.history 893 self._show_lookup(url=True) 894 self.page_index = 0 895 896 def do_search(self, searchterm): 897 """Search index (case insensitive).""" 898 results = [ 899 gi for gi in self.lookup if searchterm.lower() in gi.name.lower()] 900 if results: 901 self.lookup = results 902 self._show_lookup() 903 self.page_index = 0 904 else: 905 print("No results found.") 906 907 def emptyline(self): 908 """Page through index ten lines at a time.""" 909 i = self.page_index 910 if i > len(self.lookup): 911 return 912 self._show_lookup(offset=i, end=i+10) 913 self.page_index += 10 914 915 ### Stuff that does something to most recently viewed item 916 @needs_gi 917 def do_cat(self, *args): 918 """Run most recently visited item through "cat" command.""" 919 subprocess.call(shlex.split("cat %s" % self.tmp_filename)) 920 921 @needs_gi 922 def do_less(self, *args): 923 """Run most recently visited item through "less" command.""" 924 cmd_str = self._get_handler_cmd(self.gi) 925 cmd_str = cmd_str % self.tmp_filename 926 subprocess.call("%s | less -R" % cmd_str, shell=True) 927 928 @needs_gi 929 def do_fold(self, *args): 930 """Run most recently visited item through "fold" command.""" 931 cmd_str = self._get_handler_cmd(self.gi) 932 cmd_str = cmd_str % self.tmp_filename 933 subprocess.call("%s | fold -w 70 -s" % cmd_str, shell=True) 934 935 @needs_gi 936 def do_shell(self, line): 937 """'cat' most recently visited item through a shell pipeline.""" 938 subprocess.call(("cat %s |" % self.tmp_filename) + line, shell=True) 939 940 @needs_gi 941 def do_save(self, line): 942 """Save an item to the filesystem. 943 'save n filename' saves menu item n to the specified filename. 944 'save filename' saves the last viewed item to the specified filename. 945 'save n' saves menu item n to an automagic filename.""" 946 args = line.strip().split() 947 948 # First things first, figure out what our arguments are 949 if len(args) == 0: 950 # No arguments given at all 951 # Save current item, if there is one, to a file whose name is 952 # inferred from the gopher path 953 if not self.tmp_filename: 954 print("You need to visit an item first!") 955 return 956 else: 957 index = None 958 filename = None 959 elif len(args) == 1: 960 # One argument given 961 # If it's numeric, treat it as an index, and infer the filename 962 try: 963 index = int(args[0]) 964 filename = None 965 # If it's not numeric, treat it as a filename and 966 # save the current item 967 except ValueError: 968 index = None 969 filename = os.path.expanduser(args[0]) 970 elif len(args) == 2: 971 # Two arguments given 972 # Treat first as an index and second as filename 973 index, filename = args 974 try: 975 index = int(index) 976 except ValueError: 977 print("First argument is not a valid item index!") 978 return 979 filename = os.path.expanduser(filename) 980 else: 981 print("You must provide an index, a filename, or both.") 982 return 983 984 # Next, fetch the item to save, if it's not the current one. 985 if index: 986 last_gi = self.gi 987 try: 988 gi = self.lookup[index-1] 989 self._go_to_gi(gi, update_hist = False, handle = False) 990 except IndexError: 991 print ("Index too high!") 992 self.gi = last_gi 993 return 994 else: 995 gi = self.gi 996 997 # Derive filename from current GI's path, if one hasn't been set 998 if not filename: 999 if gi.itemtype == '1': 1000 path = gi.path 1001 if path in ("", "/"): 1002 # Attempt to derive a nice filename from the gopher 1003 # item name, falling back to the hostname if there 1004 # is no item name 1005 if not gi.name: 1006 filename = gi.host.lower() + ".txt" 1007 else: 1008 filename = gi.name.lower().replace(" ","_") + ".txt" 1009 else: 1010 # Derive a filename from the last component of the 1011 # path 1012 if path.endswith("/"): 1013 path = path[0:-1] 1014 filename = os.path.split(path)[1] 1015 else: 1016 filename = os.path.basename(gi.path) 1017 1018 # Check for filename collisions and actually do the save if safe 1019 if os.path.exists(filename): 1020 print("File %s already exists!" % filename) 1021 else: 1022 # Save the "source code" of menus, not the rendered view 1023 src_file = self.menu_filename if self.gi.itemtype in ("1", "7") else self.tmp_filename 1024 shutil.copyfile(src_file, filename) 1025 print("Saved to %s" % filename) 1026 1027 # Restore gi if necessary 1028 if index != None: 1029 self._go_to_gi(last_gi, handle=False) 1030 1031 @needs_gi 1032 def do_url(self, *args): 1033 """Print URL of most recently visited item.""" 1034 print(gopheritem_to_url(self.gi)) 1035 1036 @needs_gi 1037 def do_links(self, *args): 1038 """Extract URLs from most recently visited item.""" 1039 if self.gi.itemtype not in ("0", "h"): 1040 print("You need to visit a text item, first") 1041 return 1042 links = [] 1043 with open(self.tmp_filename, "r") as fp: 1044 for line in fp: 1045 words = line.strip().split() 1046 words_containing_urls = (w for w in words if looks_like_url(w)) 1047 urls = (extract_url(w) for w in words_containing_urls) 1048 links.extend([url_to_gopheritem(u) for u in urls]) 1049 self.lookup = links 1050 self._show_lookup() 1051 1052 ### Bookmarking stuff 1053 @needs_gi 1054 def do_add(self, line): 1055 """Add the current URL to the bookmarks menu. 1056 Bookmarks are stored in the ~/.vf1-bookmarks.txt file. 1057 Optionally, specify the new name for the bookmark.""" 1058 with open(os.path.expanduser("~/.vf1-bookmarks.txt"), "a") as fp: 1059 fp.write(gopheritem_to_line(self.gi, name=line)) 1060 1061 def do_bookmarks(self, *args): 1062 """Show the current bookmarks menu. 1063 Bookmarks are stored in the ~/.vf1-bookmarks.txt file.""" 1064 file_name = "~/.vf1-bookmarks.txt" 1065 if not os.path.isfile(os.path.expanduser(file_name)): 1066 print("You need to 'add' some bookmarks, first") 1067 else: 1068 gi = GopherItem(None, None, os.path.expanduser(file_name), 1069 "1", file_name) 1070 self._go_to_gi(gi) 1071 1072 ### Security 1073 def do_tls(self, *args): 1074 """Engage or disengage battloid mode.""" 1075 self._set_tls(not self.tls) 1076 if self.tls: 1077 print("Battloid mode engaged! Only accepting encrypted connections.") 1078 else: 1079 print("Battloid mode disengaged! Switching to unencrypted channels.") 1080 1081 ### Help 1082 def do_help(self, arg): 1083 """ALARM! Recursion detected! ALARM! Prepare to eject!""" 1084 if arg == "!": 1085 print("! is an alias for 'shell'") 1086 elif arg == "?": 1087 print("? is an alias for 'help'") 1088 else: 1089 cmd.Cmd.do_help(self, arg) 1090 1091 ### Flight recorder 1092 def do_blackbox(self, *args): 1093 """Display contents of flight recorder, showing statistics for the 1094 current gopher browsing session.""" 1095 lines = [] 1096 # Compute flight time 1097 now = time.time() 1098 delta = now - self.log["start_time"] 1099 hours, remainder = divmod(delta, 3600) 1100 minutes, seconds = divmod(remainder, 60) 1101 # Count hosts 1102 ipv4_hosts = len([host for host in self.visited_hosts if host[0] == socket.AF_INET]) 1103 ipv6_hosts = len([host for host in self.visited_hosts if host[0] == socket.AF_INET6]) 1104 # Assemble lines 1105 lines.append(("Flight duration", "%02d:%02d:%02d" % (hours, minutes, seconds))) 1106 lines.append(("Requests sent:", self.log["requests"])) 1107 lines.append((" IPv4 requests:", self.log["ipv4_requests"])) 1108 lines.append((" IPv6 requests:", self.log["ipv6_requests"])) 1109 lines.append((" TLS-secured requests:", self.log["tls_requests"])) 1110 for itemtype in sorted(self.itemtype_counts.keys()): 1111 lines.append((" Itemtype %s:" % itemtype, self.itemtype_counts[itemtype])) 1112 lines.append(("Bytes received:", self.log["bytes_recvd"])) 1113 lines.append((" IPv4 bytes:", self.log["ipv4_bytes_recvd"])) 1114 lines.append((" IPv6 bytes:", self.log["ipv6_bytes_recvd"])) 1115 lines.append(("Unique hosts visited:", len(self.visited_hosts))) 1116 lines.append((" IPv4 hosts:", ipv4_hosts)) 1117 lines.append((" IPv6 hosts:", ipv6_hosts)) 1118 lines.append(("DNS failures:", self.log["dns_failures"])) 1119 lines.append(("Timeouts:", self.log["timeouts"])) 1120 lines.append(("Refused connections:", self.log["refused_connections"])) 1121 lines.append(("Reset connections:", self.log["reset_connections"])) 1122 # Print 1123 for key, value in lines: 1124 print(key.ljust(24) + str(value).rjust(8)) 1125 1126 ### The end! 1127 def do_quit(self, *args): 1128 """Exit VF-1.""" 1129 # Clean up after ourself 1130 if self.tmp_filename: 1131 os.unlink(self.tmp_filename) 1132 if self.menu_filename: 1133 os.unlink(self.menu_filename) 1134 print() 1135 print("Thank you for flying VF-1!") 1136 sys.exit() 1137 1138 do_exit = do_quit 1139 1140 # Config file finder 1141 def get_rcfile(): 1142 rc_paths = ("~/.config/vf1/vf1rc", "~/.config/.vf1rc", "~/.vf1rc") 1143 for rc_path in rc_paths: 1144 rcfile = os.path.expanduser(rc_path) 1145 if os.path.exists(rcfile): 1146 return rcfile 1147 return None 1148 1149 # Main function 1150 def main(): 1151 1152 # Parse args 1153 parser = argparse.ArgumentParser(description='A command line gopher client.') 1154 parser.add_argument('--bookmarks', action='store_true', 1155 help='start with your list of bookmarks') 1156 parser.add_argument('--debug', action='store_true', 1157 help='start with debugging mode enabled') 1158 parser.add_argument('url', metavar='URL', nargs='*', 1159 help='start with this URL') 1160 parser.add_argument('--tls', action='store_true', 1161 help='secure all communications using TLS') 1162 parser.add_argument('--version', action='store_true', 1163 help='display version information and quit') 1164 args = parser.parse_args() 1165 1166 # Handle --version 1167 if args.version: 1168 print("VF-1 " + _VERSION) 1169 sys.exit() 1170 1171 # Instantiate client 1172 gc = GopherClient(debug=args.debug, tls=args.tls) 1173 1174 # Process config file 1175 rcfile = get_rcfile() 1176 if rcfile: 1177 print("Using config %s" % rcfile) 1178 with open(rcfile, "r") as fp: 1179 for line in fp: 1180 line = line.strip() 1181 if ((args.bookmarks or args.url) and 1182 any((line.startswith(x) for x in ("go", "g", "tour", "t"))) 1183 ): 1184 if args.bookmarks: 1185 print("Skipping rc command \"%s\" due to --bookmarks option." % line) 1186 else: 1187 print("Skipping rc command \"%s\" due to provided URLs." % line) 1188 continue 1189 gc.cmdqueue.append(line) 1190 1191 # Say hi 1192 print("Welcome to VF-1!") 1193 if args.tls: 1194 print("Battloid mode engaged! Watch your back in Gopherspace!") 1195 else: 1196 print("Enjoy your flight through Gopherspace...") 1197 1198 # Act on args 1199 if args.bookmarks: 1200 gc.cmdqueue.append("bookmarks") 1201 elif args.url: 1202 if len(args.url) == 1: 1203 gc.cmdqueue.append("go %s" % args.url[0]) 1204 else: 1205 for url in args.url: 1206 if not url.startswith("gopher://"): 1207 url = "gopher://" + url 1208 gc.cmdqueue.append("tour %s" % url) 1209 gc.cmdqueue.append("tour") 1210 1211 # Endless interpret loop 1212 while True: 1213 try: 1214 gc.cmdloop() 1215 except KeyboardInterrupt: 1216 print("") 1217 1218 if __name__ == '__main__': 1219 main()