141 lines
4.8 KiB
Python
Executable File
141 lines
4.8 KiB
Python
Executable File
import requests
|
|
from datetime import datetime
|
|
from random import choices
|
|
from threading import Thread#
|
|
from time import sleep
|
|
from math import sqrt
|
|
|
|
def do_request(url_list, count, time_delta, use_proxy=True):
|
|
global proxies
|
|
for url in choices(url_list, k=count):
|
|
kwargs = {'url':url,'proxies':proxies} if use_proxy else {'url':url}
|
|
start = datetime.now()
|
|
try:
|
|
response = requests.get(**kwargs)
|
|
except Exception as e:
|
|
print("Some error ...", e)
|
|
time_delta.append(-1)
|
|
continue
|
|
time_diff = datetime.now() - start
|
|
if response.status_code in (200, '200'):
|
|
time_delta.append(time_diff)
|
|
else:
|
|
time_delta.append(-1)
|
|
print(f"WARNING for URL '{url}'")
|
|
|
|
def do_analysis(url_list, probe_size, thread_count, use_proxy=True):
|
|
thread_timedeltas = [
|
|
[]
|
|
for _ in range(thread_count)
|
|
]
|
|
|
|
threads = [
|
|
Thread(target=do_request, args=(
|
|
url_list, probe_size, time_delta, use_proxy
|
|
))
|
|
for time_delta in thread_timedeltas
|
|
]
|
|
for thread in threads:
|
|
thread.start()
|
|
|
|
wait_string = "-\|/"
|
|
ctr = 0
|
|
while any(
|
|
thread.is_alive()
|
|
for thread in threads
|
|
):
|
|
print(f"Threads are still execution ... {wait_string[ctr%4]}\r", end="")
|
|
ctr += 1
|
|
sleep(0.5)
|
|
print("Finished!", " "*100)
|
|
|
|
for idx in range(len(threads)-1,-1,-1):
|
|
del threads[idx]
|
|
del threads
|
|
|
|
return thread_timedeltas
|
|
|
|
|
|
def do_test(url_list, probe_size, thread_count, use_proxy):
|
|
thread_timedeltas = do_analysis(url_list, probe_size, thread_count, use_proxy)
|
|
time_deltas = [
|
|
[
|
|
time.total_seconds()
|
|
for time in time_delta
|
|
if time != -1
|
|
]
|
|
for time_delta in thread_timedeltas
|
|
]
|
|
time_delta_values = [val for vals in time_deltas for val in vals]
|
|
ctr_deltas = len(time_delta_values)
|
|
min_delta = min(time_delta_values)
|
|
max_delta = max(time_delta_values)
|
|
avg_delta = sum(time_delta_values)/ctr_deltas
|
|
err_delta = (thread_count * probe_size) - ctr_deltas
|
|
std_delta = sqrt(sum( (delta-avg_delta)**2 for delta in time_delta_values)/(ctr_deltas-1))
|
|
sort_delta = sorted(time_delta_values)
|
|
med_delta = sort_delta[(ctr_deltas-1)//2]
|
|
if len(time_delta_values) % 2 == 0:
|
|
med_delta = (med_delta + sort_delta[(ctr_deltas)//2])/2
|
|
|
|
result = (min_delta, med_delta, avg_delta, max_delta, std_delta, err_delta)
|
|
return result
|
|
|
|
|
|
|
|
### INIT GET VALUES ###########################################################
|
|
proxy_file_mappings = {
|
|
#"rpi-compute": "rpi-4",
|
|
#"rpi-proxy2": "rpi-2",
|
|
"rpi-proxy0": "rpi-0",
|
|
"94.2.18.222": "rpi-pico"
|
|
}
|
|
|
|
|
|
random_links_file = "/home/lhahn/Projekte/pico-test/random_wiki_links.txt"
|
|
with open(random_links_file, "r") as file:
|
|
link_list = [
|
|
line.rstrip()
|
|
for line in file
|
|
if len(line.rstrip()) > 0
|
|
]
|
|
|
|
allfile = open("/home/lhahn/Projekte/pico-test/results_rpi-proxies.txt","a")
|
|
#allfile.write("rpi\tproxy\tprobesize\tthreads\tmin(s)\tmed(s)\tavg(s)\tmax(s)\tsd(s)\terr\n")
|
|
first_round = False
|
|
### ANALYSIS DONE IN SEVERAL CONFIGS ##########################################
|
|
for proxy,filename in proxy_file_mappings.items():
|
|
proxies = {
|
|
'http': f'http://{proxy}:8080',
|
|
'https': f'http://{proxy}:8080'
|
|
}
|
|
with open(f"/home/lhahn/Projekte/pico-test/results_{filename}-proxy.txt", "w") as f:
|
|
f.write("proxy\tprobesize\tthreads\tmin(s)\tmed(s)\tavg(s)\tmax(s)\tsd(s)\terr\n")
|
|
for use_proxy in (False, True):
|
|
for probe_size in (10,25,50,100):
|
|
for thread_count in (1,5,10,20):
|
|
min_delta, med_delta, avg_delta, max_delta, std_delta, err_delta = do_test(
|
|
link_list, probe_size, thread_count, use_proxy
|
|
)
|
|
data_str = (
|
|
f"{use_proxy}\t{probe_size}\t{thread_count}\t{min_delta}\t{med_delta}"
|
|
f"\t{avg_delta}\t{max_delta}\t{std_delta}\t{err_delta}\n"
|
|
)
|
|
result_str = (
|
|
f"avg: {avg_delta}s, med:{med_delta} min: {min_delta}s, max: {max_delta}s, "
|
|
f"std: {std_delta}s | err: {err_delta}/{probe_size*thread_count} | "
|
|
f"thrds: {thread_count}, proxy: {use_proxy}, probe_size: {probe_size}, "
|
|
f"rpi: {filename}"
|
|
)
|
|
print(result_str)
|
|
|
|
f.write(data_str)
|
|
f.flush()
|
|
if use_proxy or (not use_proxy and first_round):
|
|
allfile.write(
|
|
f"{filename}\t{data_str}"
|
|
)
|
|
allfile.flush()
|
|
first_round = False
|
|
allfile.close()
|