Wikipedia indexer and shortest link finder I have the following code, how can I make it more efficient? Also, it doesn't always find the shortest route. (See Cat -> Tree)

import urllib.request, re, json

try:
    file = open("urls.json", "r")
    urls = json.loads(file.read())
    file.close()

    file = open("links.json", "r")
    links = json.loads(file.read())
    file.close()
except:
    urls = ["http://en.wikipedia.org/wiki/Cat"]
    links = [[-1, 0]]

for url in urls:
    try:
        print("Trying: " + url)
        request = urllib.request.Request(url, headers = {"User-Agent" : "DanBot/2.0"})
        file = urllib.request.urlopen(request)
        source = file.read().decode("utf-8")
    except Exception as e:
        print("Failed: " + url)
        while url in urls:
            urls.remove(url)

    suburls = re.findall("href=[\"']?([^\"' &gt;]+)", source)

    for suburl in suburls:
        if len(suburl) &gt; 6 and suburl[:7] == "http://" or suburl[:8] == "https://":
            fullurl = suburl
        elif len(suburl) &gt; 1 and suburl[:2] == "//":
            fullurl = "http:" + suburl
        elif len(suburl) &gt; 0 and suburl[0] == "/":
            fullurl = "/".join(url.split("/")[:3]) + suburl
        else:
            fullurl = "/".join(url.split("/")[:-1]) + "/" + suburl

        if fullurl[:29].lower() == "http://en.wikipedia.org/wiki/" and "#" not in fullurl[29:] and ":" not in fullurl[29:]:
            if fullurl not in urls:
                urls.append(fullurl)
            if [urls.index(url), urls.index(fullurl)] not in links:
                links.append([urls.index(url), urls.index(fullurl)])
        else:
            continue

    file = open("urls.json", "w")
    file.write(json.dumps(urls))
    file.close()

    file = open("links.json", "w")
    file.write(json.dumps(links))
    file.close()

input()


That program creates the index, and this one finds the links:

import urllib.request, re, json

try:
    file = open("urls.json", "r")
    urls = json.loads(file.read())
    file.close()

    file = open("links.json", "r")
    links = json.loads(file.read())
    file.close()
except:
    print("Error: No link or url file found")
    input()
    exit()

starturl = "http://en.wikipedia.org/wiki/" + input("Start at: http://en.wikipedia.org/wiki/")
endurl = "http://en.wikipedia.org/wiki/" + input("End at: http://en.wikipedia.org/wiki/")

if starturl not in urls or endurl not in urls:
    print("Error: One or more urls not found in index")
    input()
    exit()

print("")

route = []
selection = [[-1, urls.index(starturl)]]

done = False
for link in selection:
    for link2 in links:
        if link[1] == link2[0]:
            selection.append(link2)
        if link2[1] == urls.index(endurl) and link2 in selection:
            done = True
            break
    if done:
        break

if not done:
    print("Error: No link found between pages")
    input()
    exit()

selection = list(reversed(selection))
route = [selection[0]]
for link in selection:
    if link[1] == route[-1][0]:
        route.append(link)
route = list(reversed(route))

print("Route:")
for link in route:
    print(urls[link[1]])

input()
** this won't make your code more efficient, but is still something you should consider doing

With all your file opening, you should do it as:

with open('filename','r') as fin:
    urls/links = json.loads(fin.read())


while you have the try/catch block, your exception handler is not checking and closing open file handles, which is what the with context does