Hello everyone, I am developing a geolocation application that receives connection accesses to a server in real time, a constant log of Apache IPs is generated and from this constant log flow I use GEOIP2 to process this data and convert it into a json that would be sent to my front endthrough websocket using channels, which would show me a real-time world map similar to the bit defender threatmap: https://threatmap.bitdefender.com.
I created a function that simulates a tail, but I realized that when placing it in the view I have a problem in which the front end does not load.
def tail_apache_log(file_path, namefiles):
print("COMEÇANDO TAILL")
time_pattern = re.compile(r'\d{2}/\w{3}/\d{4}:\d{2}:\d{2}:\d{2}')
ip_pattern = re.compile(r'\d+\.\d+\.\d+\.\d+')
result = f"{file_path}{namefiles}.log"
with open(result, 'r', encoding='utf-8') as log_file:
log_file.seek(0, 2) # Vá para o final do arquivo (modo append)
while True:
line = log_file.readline()
if not line:
asyncio.sleep(1)
continue
time_match = time_pattern.search(line)
ip_match = ip_pattern.search(line)
if time_match and ip_match:
log_time = time_match.group(0)
log_ip = ip_match.group(0)
data_hora = converter_string_para_data(log_time)
if log_ip.startswith("10.") or log_ip.startswith("172."):
#print(f"ip interno {log_ip}")
pass
else:
lat, long, loc_country, location_city = getGeoIpData(log_ip)
#print(log_ip)
data_to_send = {'ip': log_ip, 'lat': (lat), 'long': long,'data': str(data_hora), 'ip': log_ip, 'country': loc_country, 'city': location_city}
#ENVIAR DADOS E DEPOIS VOLTAR NO LOOP
#ws_map.enviar_dados(data_to_send)
print(data_to_send)
I also thought about the alternative of calling the function in the consumer itself but I saw that this is not a good practice and it was also consuming a lot of CPU
class MapaConsumer(AsyncWebsocketConsumer):
async def connect(self):
self.groupname = 'Pontos_Mapa'
await self.channel_layer.group_add(
self.groupname,
self.channel_name,
)
log_thread = threading.Thread(target=self.tail_apache_log_thread)
log_thread.daemon = True
log_thread.start()
await self.accept()
async def receive(self, text_data):
await self.channel_layer.group_send(
self.groupname,
{
'type': 'json_objects_tunel',
'value': text_data,
}
)
async def json_objects_tunel(self, event):
print(event['value'])
await self.send(event['value'])
async def disconnect(self, close_code):
print("\n =====CONSUMER DESCONECTADO======")
print(f"Conexão Fechada: {close_code}\n")
# pass
await self.channel_layer.group_discard(
self.groupname,
self.channel_name,
)
I thought about doing the function asynchronously but I’m not sure if there is a better alternative with Kafka (I’ve never used it before) or Celery. Can anyone get a light? or if I run tail asynchronously can I do it?
My view and class to create websocket connection
def home2(request):
#tail_apache_log(path,nome_arquivo)
criando_classe_WS()
context = {}
return render(request, "site/index.html", context)
Class for connection
import websocket
import json
conections_list = []
def criando_classe_WS():
ws1 = Conexao()
ws1.criarConexao_Mapa()
return ws1
class Conexao():
def __init__(self):
self.ws_obj = websocket.WebSocket()
def criarConexao_Mapa(self):
self.ws_obj.connect('ws://localhost:8000/ws/tables/')
return self.ws_obj
def send_data(self, data_to_send):
#print(data_to_send)
self.ws_obj.send(json.dumps(data_to_send))
