C# kndlCrawler (crawling without account)

[aspire]

Hi,

mein c# crawler für knuddels mit ein paar schönen console highlights.
Crawlt alle nicks aus jedem channel und speichert sie falls gewünscht anschließen in eine .txt.

Crawler.cs

using System;

using System.Collections.Generic;

using System.Linq;

using System.Text;

using System.Threading;

using System.Text.RegularExpressions;

using System.IO;

using System.Diagnostics;

using mLib;



namespace kndlCrawler

{

  class Program

  {

    private static readonly Regex channelListParse = new Regex("\0([a-zA-Z0-9+!\\-" ]+?)\n([0-9]+?)\0", RegexOptions.Compiled);

    private static readonly Regex nickParse = new Regex("°>_h(.+?)\\|/serverpp "\\|/w "<°", RegexOptions.Compiled);



    private static AutoResetEvent autoEventPopup;

    private static AutoResetEvent autoEventConnected;

    private static AutoResetEvent autoEventCrawling;



    private static int subChanCount = 1;

    private static Client client = new Client();



    private static Channel lastCrawledChan = new Channel();

    private static List<Channel> channels = new List<Channel>();

    private static List<Channel> crawledChannels = new List<Channel>();

    private static List<Nickname> crawledNicknames = new List<Nickname>();



    static void Main(string[] args)

    {

      Console.WriteLine("Choose chatsystem:\n");

      RemoteEndpoint remoteEndPoint = GetRemoteEndPointInput();

      ConsoleStuff.ClearLastConsoleLines(2);

      Console.CursorVisible = false;

      Console.WriteLine("Chatsystem: {0}", remoteEndPoint);

      Console.WriteLine("Connecting to {0}:{1} [{2}]..", Client.GetHost(remoteEndPoint), Client.GetPort(remoteEndPoint), remoteEndPoint);



      Stopwatch sw = new Stopwatch();

      sw.Start();



      autoEventCrawling = new AutoResetEvent(false);

      Thread waitThread = ConsoleStuff.PrintWait();

      client.onReceive += OnReceive;

      client.Connect(remoteEndPoint);

      autoEventCrawling.WaitOne();

      waitThread.Abort();



      sw.Stop();

      Console.WriteLine("Crawled {0} nicknames in {1} channels in {2} seconds. :)", crawledNicknames.Count, channels.Count, sw.ElapsedMilliseconds / 1000);



      string saveContent = String.Empty;

      if (ConsoleStuff.YesNoAsk("Save crawled nicknames?  ["crawlednicknames.txt"]"))

      {

        foreach (Nickname nickname in crawledNicknames)

        {

            saveContent += String.Format("{0} | {1}\n", nickname.Name, nickname.Channel);

        }

        File.WriteAllText("crawlednicknames.txt", saveContent);

      }



      saveContent = String.Empty;

      if (ConsoleStuff.YesNoAsk("Save crawled channels? ["crawledchannels.txt"]"))

      {

        foreach (Channel channel in crawledChannels)

        {

          saveContent += String.Format("{0} | {1} | {2}\n", channel.Name, channel.UserCount, channel.Nicknames.Count);

        }

        File.WriteAllText("crawledchannels.txt", saveContent);

      }

      Console.WriteLine("Done. :)");

      Console.ReadLine();

    }



    private static void OnReceive(Client client, string token)

    {

      string[] tokens = token.Split('\0');



      switch (tokens[0])

      {

        case "b":

          if (autoEventConnected != null)

            autoEventConnected.Set();



          if (channels.Count == 0)

          {

            channels = ParseChannels(token);

            new Thread(delegate() { CrawlNicks(client); }).Start();

          }

          break;



        case "k":

          if (tokens[1].StartsWith(lastCrawledChan.Name))

          {

            List<Nickname> parsedNicks = new List<Nickname>();

            parsedNicks.AddRange(ParseNicks(tokens[7]));



            crawledNicknames.AddRange(parsedNicks);

            lastCrawledChan.Nicknames = parsedNicks;

            crawledChannels.Add(lastCrawledChan);

            autoEventPopup.Set();

          }

          break;

      }

    }



    private static void CrawlNicks(Client client)

    {

      foreach (Channel channel in channels)

      {

        Console.Write("Crawling channel "{0}".. ", channel.Name);

        while (!crawledChannels.Contains(channel))

        {

          if (!client.IsConnected())

          {

            client.ReConnect();

            autoEventConnected = new AutoResetEvent(false);

            autoEventConnected.WaitOne();

          }



          client.Send(String.Format("r\0{0}", channel.Name));

          lastCrawledChan = channel;

          autoEventPopup = new AutoResetEvent(false);

          autoEventPopup.WaitOne(2222);

        }

        Console.Write("Crawled {0} nicks.\n", lastCrawledChan.Nicknames.Count);

      }



      autoEventCrawling.Set();

    }



    private static List<Nickname> ParseNicks(string nicksstring)

    {

      List<Nickname> retVal = new List<Nickname>();

      foreach (Match m in nickParse.Matches(nicksstring))

      {

        if (m.Success && m.Groups.Count == 2 && !String.IsNullOrEmpty(m.Groups[1].Value))

        {

          if (m.Groups[1].Value != client.Butler)

            retVal.Add(new Nickname(m.Groups[1].Value.Replace("\", ""), lastCrawledChan.Name));

        }

      }



      return retVal;

    }



    private static List<Channel> ParseChannels(string token)

    {

      List<Channel> retVal = new List<Channel>();



      foreach (Match m in channelListParse.Matches(token))

      {

        if (m.Success && m.Groups.Count == 3)

        {

          string channame = m.Groups[1].Value;



          if (!string.IsNullOrEmpty(channame))

          {

            if (channame == """)

            {

              channame = String.Format("{0} {1}", retVal.ElementAt(retVal.Count - subChanCount).Name, ++subChanCount);

            }

            else

            {

              subChanCount = 1;

            }

            retVal.Add(new Channel(channame, int.Parse(m.Groups[2].Value), new List<Nickname>()));

          }

        }

      }



      return retVal;

    }



    private static RemoteEndpoint GetRemoteEndPointInput()

    {

      List<string> retVal = new List<string>();

      string[] remoteEndPoints = Enum.GetNames(typeof(RemoteEndpoint));

      for (int i = 0; i < remoteEndPoints.Length; i++)

      {

        retVal.Add(String.Format("{0} [{1}:{2}]", remoteEndPoints[i], Client.GetHost((RemoteEndpoint)i), Client.GetPort((RemoteEndpoint)i)));

      }



      return (RemoteEndpoint)ConsoleStuff.PrintMenu(retVal);

    }





  }

}

ConsoleStuff.cs

using System;

using System.Collections.Generic;

using System.Linq;

using System.Text;

using System.Threading;



namespace kndlCrawler

{

  class ConsoleStuff

  {

    public static bool YesNoAsk(string question)

    {

      ConsoleKeyInfo key = new ConsoleKeyInfo();

      while (key.Key != ConsoleKey.Y && key.Key != ConsoleKey.N)

      {

        Console.Write("{0} (Y/N) ", question);

        key = Console.ReadKey();

        Console.Write("\n");

      }

      return key.Key == ConsoleKey.Y;

    }



    public static int PrintMenu(List<string> menuItems)

    {

      ConsoleKeyInfo key = new ConsoleKeyInfo();

      Console.TreatControlCAsInput = true;

      Console.CursorVisible = false;



      int selected = 0;



      while (key.Key != ConsoleKey.Enter)

      {

        if (key.Key == ConsoleKey.UpArrow && selected == 0)

          selected = menuItems.Count;

        if (key.Key == ConsoleKey.DownArrow && selected == menuItems.Count - 1)

          selected = -1;

        if (key.Key == ConsoleKey.UpArrow && selected > 0)

          selected -= 1;

        if (key.Key == ConsoleKey.DownArrow && selected < menuItems.Count - 1)

          selected += 1;



        for (int i = 0; i < menuItems.Count; i++)

        {

          if (i == selected)

          {

            Console.WriteLine("[x] {0}", menuItems[i]);

          }

          else

          {

            Console.WriteLine("[ ] {0}", menuItems[i]);

          }

        }

        key = Console.ReadKey();

        ClearLastConsoleLines(menuItems.Count);

      }



      Console.CursorVisible = true;

      return selected;

    }



    public static void ClearLastConsoleLines(int count)

    {

      int cursorTop = Console.CursorTop;



      if (count == cursorTop || count == 0)

      {

        Console.Clear();

      }

      else if (count < cursorTop && count > 0)

      {

        string clearString = new String(' ', Console.WindowWidth * count);



        Console.SetCursorPosition(0, cursorTop - count);

        Console.Write(clearString);

        Console.SetCursorPosition(0, cursorTop - count);

      }

    }



    public static Thread PrintWait()

    {

      Thread t = new Thread(delegate()

      {

        char[] waitstring = new char[] { '|', '/', '-', '\\' };

        int charcount = 0;

        while (true)

        {

          Console.Write(waitstring[charcount]);

          Console.SetCursorPosition(Console.CursorLeft - 1, Console.CursorTop);

          if (charcount + 1 >= waitstring.Length)

            charcount = 0;

          else

            charcount++;

          Thread.Sleep(100);

        }

      });

      t.Start();

      return t;

    }

  }

}

Channel.cs

using System;

using System.Collections.Generic;

using System.Linq;

using System.Text;



namespace kndlCrawler

{

  class Channel

  {

    public string Name { get; set; }

    public int UserCount { get; set; }

    public List<Nickname> Nicknames { get; set; }



    public Channel()

    {



    }



    public Channel(string name, int userCount, List<Nickname> nicks)

    {

      Name = name;

      UserCount = userCount;

      Nicknames = nicks;

    }

  }

}

Nickname.cs

using System;

using System.Collections.Generic;

using System.Linq;

using System.Text;



namespace kndlCrawler

{

  class Nickname

  {

    public string Name { get; set; }

    public string Channel { get; set; }



    public Nickname(string name, string channel)

    {

      Name = name;

      Channel = channel;

    }

  }

}

Screens von der Console:



Hier lässt sich mit den CursorTasten ein Chatsystem aus dem RemoteEndPoint Enum auswählen.



Beim crawlen.



Nach dem crawlen.

Das ganze läuft mit einer "mLib" - die KNLib nur für MFC optimiert.
Mit einer richtigen applet.jar funktioniert es auch wunderbar mit knuddels.
Wegen der KNLib natürlich ein big thanks an flav.

Hier das ganze als download (mLib.dll + kndlCrawler.exe mit m90aca als applet - die aktuelle mfc version):

Klick

Have fun.

-aspire

[Flav]

Soweit ich weiß muss man da keine besonderen Änderungen vornehmen damit die KNLib für den MFC optimiert ist, einfach eine andere applet.jar verwenden.

[aspire]

@Flav

ja stimmt schon - ich hab halt weng was umgecoded..
z.b. das mit den getHost und getPort von den RemoteEndPoints ist public das man in der Console den host anzeigen kann..
Dann alle Events reingecoded die so ca. alles vorparsen..
dann beim einloggsystem der jedes problem handelt (banned, falsches pw, chan und so)
dazu nen event wann man eingeloggt & ausgeloggt ist..

aber vom gerüst des codes ists immer noch deine KNLib - deswegen credits an dich.

[Flav]

Um fehlgeschlagene Logins zu behandeln würde ich einfach einen Popup Parser schreiben, ist ja schon größtenteils Open Source.

[aspire]

@Flav

es ist ja quasi in popup parser - nur eben schon in der lib drinnen der die popups handelt wie z.b. problem wegen faslcherm nickname, dann gibt er das geparste noch an ein onPopup event mit entsprechenden paramtern weiter - das ist das handling von felhgeschlagenen logins in der lib

/e: aber darum gehts ja eig. nicht bei dem source..