Subversion Repositories general

Rev

Rev 1241 | Blame | Compare with Previous | Last modification | View Log | RSS feed

using System;
using System.Collections;
using System.IO;
using System.Net;
using System.Net.Sockets;
using System.Threading;
using System.Text;
using System.Text.RegularExpressions;
using System.Xml;

// FIXME:
//   - deny write access to all public properties
//   - for text/xml get encoding from body if not specified in header
//   - option to not store parsed data, just raw packets and reparse on demand
//   - implement all others (not 'identity' and 'chunked') transfer- and content- encodings
//   - benchmark one-by-one byte iterator vs. block read
namespace TCPproxy
{
        public enum BinLogTypes
        {
                None = 1,
                TcpConnection
        }

        public enum LogLevel
        {
                Critical,
                Error,
                Warning,
                Important,
                Info,
                Debug
        }

        public enum SocketState
        {
                None,
                Connecting,
                Connected,
                ShutdownSend,
                ShutdownReceived,
                Closed
        }

        public enum TcpMessageDirection
        {
                None,
                Local,
                Remote
        }

        public enum HttpVersion
        {
                V0_9,
                V1_0,
                V1_1
        }

        public enum HttpTransferEncoding
        {
                None,              // there is no body
                Identity,
                Chunked,
                Gzip,
                Compress,
                Deflate,
                Unknown
        }

        public enum HttpContentEncoding
        {
                None,              // there is no body
                Identity,
                Gzip,
                Compress,
                Deflate,
                Unknown
        }

        public class TcpLogEventArgs : EventArgs
        {
                private readonly LogLevel  level;
                private readonly string    message;
                private readonly Exception exception;

                public LogLevel Level
                {
                        get { return level; }
                }

                public string Message
                {
                        get { return message; }
                }

                public Exception Exception
                {
                        get { return exception; }
                }

                internal TcpLogEventArgs(LogLevel level, string message, Exception exception)
                {
                        this.level     = level;
                        this.message   = message;
                        this.exception = exception;
                }
        }

        public class TcpEventArgs : EventArgs
        {
                internal TcpEventArgs()
                {
                }
        }

        public class TcpConnectionEventArgs : EventArgs
        {
                private readonly TcpConnection tcp;

                public TcpConnection Tcp
                {
                        get { return tcp; }
                }

                internal TcpConnectionEventArgs(TcpConnection tcp)
                {
                        this.tcp = tcp;
                }
        }

        public class TcpHttpEventArgs : EventArgs
        {
                private readonly HttpMessage http;

                public HttpMessage Http
                {
                        get { return http; }
                }

                internal TcpHttpEventArgs(HttpMessage http)
                {
                        this.http = http;
                }
        }

        public delegate void TcpLogEventHandler(object sender, TcpLogEventArgs e);

        public delegate void TcpEventHandler(object sender, TcpEventArgs e);

        public delegate void TcpConnectionEventHandler(object sender, TcpConnectionEventArgs e);

        public delegate void TcpHttpEventHandler(object sender, TcpHttpEventArgs e);

        public class TcpListener
        {
                private int       listenPort     = -1;
                private IPAddress resendHost;
                private int       resendPort     = -1;
                private Socket    socket;
                private int       tcpId          = 0;
                private ArrayList tcpConnections = new ArrayList();

                public TcpListener(int listenPort, IPAddress resendHost, int resendPort)
                {
                        this.listenPort = listenPort;
                        this.resendHost = resendHost;
                        this.resendPort = resendPort;
                }

                public void StartListening()
                {
                        socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
                        socket.Bind(new IPEndPoint(IPAddress.Any, listenPort));
                        socket.Listen(100);
                        socket.BeginAccept(new AsyncCallback(OnClientConnect), null);
                        SendLog(null, LogLevel.Important, "Listen on " + socket.LocalEndPoint);
                }

                public void StopListening()
                {
                        try
                        {
                                if(socket != null)
                                {
                                        SendLog(null, LogLevel.Important, "Stop listening " + socket.LocalEndPoint);
                                        socket.Close();
                                }
                        }
                        catch(ObjectDisposedException) // socket is already closed
                        {
                        }
                        catch(Exception ex)
                        {
                                Console.WriteLine(ex.Message + " (" + ex.GetType().Name + ")\n" + ex.StackTrace);
                        }
                }

                public void CancelAll()
                {
                        // make a copy of connections list because it will be changed when connections will delete themself
                        ArrayList tcpConnectionsCopy;
                        lock(tcpConnections)
                        {
                                tcpConnectionsCopy = new ArrayList(tcpConnections);
                        }

                        // send cancel
                        foreach(TcpConnection tcp in tcpConnectionsCopy)
                        {
                                tcp.Cancel();
                        }

                        // wait for aborted threads
                        foreach(TcpConnection tcp in tcpConnectionsCopy)
                        {
                                tcp.WaitCancel();
                        }
                }

                protected virtual void OnClientConnect(IAsyncResult asyn)
                {
                        try
                        {
                                Socket worker = socket.EndAccept(asyn);
                                socket.BeginAccept(new AsyncCallback(OnClientConnect), null); // wait for next client

                                TcpConnection tcp = new TcpConnection(string.Format("{0:0000}", tcpId++));
                                tcp.Close += new TcpEventHandler(TcpConnectionClosed);
                                OnNewTcp(new TcpConnectionEventArgs(tcp));

                                lock(tcpConnections)
                                {
                                        tcpConnections.Add(tcp);
                                }

                                tcp.Continue(resendHost, resendPort, worker);
                        }
                        catch(ObjectDisposedException) {}  // socket is closed
                        catch(Exception ex)
                        {
                                Console.WriteLine(ex.Message + " (" + ex.GetType().Name + ")\n" + ex.StackTrace);
                        }
                }

                protected virtual void TcpConnectionClosed(object sender, TcpEventArgs e)
                {
                        lock(tcpConnections)
                        {
                                tcpConnections.Remove((TcpConnection)sender);
                        }
                }

                public event TcpConnectionEventHandler NewTcp;

                protected virtual void OnNewTcp(TcpConnectionEventArgs e)
                {
                        if(NewTcp != null)
                        {
                                NewTcp(this, e);
                        }
                }

                public event TcpLogEventHandler Log;

                protected virtual void OnLog(TcpLogEventArgs e)
                {
                        if(Log != null)
                        {
                                Log(this, e);
                        }
                }

                protected virtual void SendLog(TcpConnection tcp, LogLevel level, string message)
                {
                        TcpLogEventArgs e = new TcpLogEventArgs(level, message, null);
                        OnLog(e);
                }

                protected virtual void SendLog(TcpConnection tcp, LogLevel level, Exception ex)
                {
                        TcpLogEventArgs e = new TcpLogEventArgs(level, null, ex);
                        OnLog(e);
                }

                public void ReadBinLog(BinaryReader reader)
                {
                        // tcp connections
                        TcpConnection tcp;
                        while((tcp = TcpConnection.ReadBinLog(reader)) != null) {
                                OnNewTcp(new TcpConnectionEventArgs(tcp));
                        }
                }
        }

        public class TcpConnection
        {
                private string             id;
                private DateTime           startTimestamp     = DateTime.MinValue;
                private DateTime           localEndTimestamp  = DateTime.MinValue;
                private DateTime           remoteEndTimestamp = DateTime.MinValue;
                private IPEndPoint         localPoint;
                private IPEndPoint         remotePoint;
                private LinkedList         messages           = new LinkedList();
                private LinkedListReadOnly messagesReadOnly;
                private SocketState        localState         = SocketState.None;
                private SocketState        remoteState        = SocketState.None;
                private SocketWorker       worker;
                private LinkedList         https              = new LinkedList();
                private HttpParser         httpParser;

                public string Id
                {
                        get { return id; }
                }

                public DateTime StartTimestamp
                {
                        get { return startTimestamp; }
                }

                public DateTime LocalEndTimestamp
                {
                        get { return localEndTimestamp; }
                }

                public DateTime RemoteEndTimestamp
                {
                        get { return remoteEndTimestamp; }
                }

                public IPEndPoint LocalPoint
                {
                        get { return localPoint; }
                }

                public IPEndPoint RemotePoint
                {
                        get { return remotePoint; }
                }

                public LinkedListReadOnly Messages
                {
                        get { return new LinkedListReadOnly(messages); }
                }

                public SocketState LocalState
                {
                        get { return localState; }
                }

                public SocketState RemoteState
                {
                        get { return remoteState; }
                }

                private int CountBytes(TcpMessageDirection direction)
                {
                        int count = 0;

                        foreach(TcpMessage message in messages)
                        {
                                if(message.Direction == direction)
                                        count += message.Length;
                        }
                        return count;
                }

                public int SentBytes
                {
                        get { return CountBytes(TcpMessageDirection.Local); }
                }

                public int ReceivedBytes
                {
                        get { return CountBytes(TcpMessageDirection.Remote); }
                }

                internal TcpConnection(string id)
                {
                        this.id               = id;
                        this.httpParser       = HttpParser.Parse(this);
                        this.messagesReadOnly = new LinkedListReadOnly(messages);
                }

                internal void Continue(IPAddress resendHost, int resendPort, Socket localSocket)
                {
                        SendLog(LogLevel.Important, "Client connected from " + ((IPEndPoint)localSocket.RemoteEndPoint).ToString());
                        SetLocalState(SocketState.Connected);
                        this.worker = new SocketWorker(resendHost, resendPort, localSocket, this);
                }

                public void Cancel()
                {
                        if(worker != null)
                                worker.Cancel();
                }

                public void WaitCancel()
                {
                        if(worker != null)
                                worker.WaitCancel();
                }

                protected TcpMessage Append(TcpMessageDirection direction, byte[] newBytes)
                {
                        return Append(direction, newBytes, newBytes.Length);
                }

                protected TcpMessage Append(TcpMessageDirection direction, byte[] newBytes, int length)
                {
                        if(newBytes == null) return null;

                        TcpMessage message;

                        lock(this)
                        {
                                message = new TcpMessage();
                                message.Direction = direction;
                                messages.Add(message);
                                message.Append(newBytes, length);
                        }

                        httpParser.NewMessageArived();
                        OnUpdate(new TcpEventArgs());

                        return message;
                }

                internal void AddHttpMessage(HttpMessage http)
                {
                        lock(this)
                        {
                                https.Add(http);
                                TcpHttpEventArgs e = new TcpHttpEventArgs(http);
                                OnNewHttp(e);
                        }
                }

        protected void SetLocalPoint(IPEndPoint localPoint)
                {
                        this.localPoint = localPoint;
                        OnUpdate(new TcpEventArgs());
                }

                protected void SetRemotePoint(IPEndPoint remotePoint)
                {
                        this.remotePoint = remotePoint;
                        OnUpdate(new TcpEventArgs());
                }

                private void CheckClosed()
                {
                        if(localState == SocketState.Closed && remoteState == SocketState.Closed)
                                OnClose(new TcpEventArgs());
                }

                protected void SetLocalState(SocketState localState)
                {
                        if(this.localState == SocketState.None && localState == SocketState.Connecting)
                        {
                                startTimestamp = DateTime.Now;
                        }
                        else if(this.localState == SocketState.None && localState == SocketState.Connected)
                        {
                                startTimestamp = DateTime.Now;
                        }
                        else if(this.localState == SocketState.None && localState == SocketState.Closed)
                        {
                        }
                        else if(this.localState == SocketState.Connecting && localState == SocketState.Connected)
                        {
                        }
                        else if(this.localState == SocketState.Connecting && localState == SocketState.Closed)
                        {
                                if(localEndTimestamp == DateTime.MinValue) localEndTimestamp = DateTime.Now;
                        }
                        else if(this.localState == SocketState.Connected && localState == SocketState.ShutdownSend)
                        {
                        }
                        else if(this.localState == SocketState.Connected && localState == SocketState.ShutdownReceived)
                        {
                                if(localEndTimestamp == DateTime.MinValue) localEndTimestamp = DateTime.Now;
                        }
                        else if(this.localState == SocketState.Connected && localState == SocketState.Closed)
                        {
                                if(localEndTimestamp == DateTime.MinValue) localEndTimestamp = DateTime.Now;
                        }
                        else if(this.localState == SocketState.ShutdownSend && localState == SocketState.Closed)
                        {
                                if(localEndTimestamp == DateTime.MinValue) localEndTimestamp = DateTime.Now;
                        }
                        else if(this.localState == SocketState.ShutdownSend && localState == SocketState.ShutdownReceived)
                        {
                                if(localEndTimestamp == DateTime.MinValue) localEndTimestamp = DateTime.Now;
                        }
                        else if(this.localState == SocketState.ShutdownReceived && localState == SocketState.ShutdownSend)
                        {
                        }
                        else if(this.localState == SocketState.ShutdownReceived && localState == SocketState.Closed)
                        {
                                if(localEndTimestamp == DateTime.MinValue) localEndTimestamp = DateTime.Now;
                        }
                        else if(this.localState == SocketState.Closed && localState == SocketState.Closed)
                        {
                                if(localEndTimestamp == DateTime.MinValue) localEndTimestamp = DateTime.Now;
                        }
                        else
                        {
                                throw new Exception("Wrong local socket state change: from " + this.localState + " to " + localState);
                        }
                        this.localState = localState;
                        if(this.localState == SocketState.Closed) httpParser.NewMessageArived();
                        OnUpdate(new TcpEventArgs());
                        CheckClosed();
                }

                protected void SetRemoteState(SocketState remoteState)
                {
                        if(this.remoteState == SocketState.None && remoteState == SocketState.Connecting)
                        {
                        }
                        else if(this.remoteState == SocketState.None && remoteState == SocketState.Connected)
                        {
                        }
                        else if(this.remoteState == SocketState.None && remoteState == SocketState.Closed)
                        {
                        }
                        else if(this.remoteState == SocketState.Connecting && remoteState == SocketState.Connected)
                        {
                        }
                        else if(this.remoteState == SocketState.Connecting && remoteState == SocketState.Closed)
                        {
                                if(remoteEndTimestamp == DateTime.MinValue) remoteEndTimestamp = DateTime.Now;
                        }
                        else if(this.remoteState == SocketState.Connected && remoteState == SocketState.ShutdownSend)
                        {
                        }
                        else if(this.remoteState == SocketState.Connected && remoteState == SocketState.ShutdownReceived)
                        {
                                if(remoteEndTimestamp == DateTime.MinValue) remoteEndTimestamp = DateTime.Now;
                        }
                        else if(this.remoteState == SocketState.Connected && remoteState == SocketState.Closed)
                        {
                                if(remoteEndTimestamp == DateTime.MinValue) remoteEndTimestamp = DateTime.Now;
                        }
                        else if(this.remoteState == SocketState.ShutdownSend && remoteState == SocketState.Closed)
                        {
                                if(remoteEndTimestamp == DateTime.MinValue) remoteEndTimestamp = DateTime.Now;
                        }
                        else if(this.remoteState == SocketState.ShutdownSend && remoteState == SocketState.ShutdownReceived)
                        {
                                if(remoteEndTimestamp == DateTime.MinValue) remoteEndTimestamp = DateTime.Now;
                        }
                        else if(this.remoteState == SocketState.ShutdownReceived && remoteState == SocketState.ShutdownSend)
                        {
                        }
                        else if(this.remoteState == SocketState.ShutdownReceived && remoteState == SocketState.Closed)
                        {
                                if(remoteEndTimestamp == DateTime.MinValue) remoteEndTimestamp = DateTime.Now;
                        }
                        else if(this.remoteState == SocketState.Closed && remoteState == SocketState.Closed)
                        {
                                if(remoteEndTimestamp == DateTime.MinValue) remoteEndTimestamp = DateTime.Now;
                        }
                        else
                        {
                                throw new Exception("Wrong remote socket state change: from " + this.remoteState + " to " + remoteState);
                        }
                        this.remoteState = remoteState;
                        if(this.remoteState == SocketState.Closed) httpParser.NewMessageArived();
                        OnUpdate(new TcpEventArgs());
                        CheckClosed();
                }

                public event TcpEventHandler Update;

                protected virtual void OnUpdate(TcpEventArgs e)
                {
                        if(Update != null)
                        {
                                Update(this, e);
                        }
                }

                public event TcpHttpEventHandler NewHttp;

                protected virtual void OnNewHttp(TcpHttpEventArgs e)
                {
                        if(NewHttp != null)
                        {
                                NewHttp(this, e);
                        }
                }

                public event TcpEventHandler Close;

                protected virtual void OnClose(TcpEventArgs e)
                {
                        if(Close != null)
                        {
                                Close(this, e);
                        }
                }

                public event TcpLogEventHandler Log;

                protected virtual void OnLog(TcpLogEventArgs e)
                {
                        if(Log != null)
                        {
                                Log(this, e);
                        }
                }

                protected virtual void SendLog(LogLevel level, string message)
                {
                        TcpLogEventArgs e = new TcpLogEventArgs(level, message, null);
                        OnLog(e);
                }

                protected virtual void SendLog(LogLevel level, Exception ex)
                {
                        TcpLogEventArgs e = new TcpLogEventArgs(level, null, ex);
                        OnLog(e);
                }

                private void WriteEndPoint(BinaryWriter writer, IPEndPoint endPoint)
                {
                        // end point as (family as int, ip length, ip as bytes, port)
                        byte[] ipBuf = endPoint.Address.GetAddressBytes();
                        writer.Write((UInt32)endPoint.AddressFamily);
                        writer.Write((UInt32)ipBuf.Length);
                        writer.Write(ipBuf);
                        writer.Write((UInt32)endPoint.Port);
                }

                public void WriteBinLog(BinaryWriter writer)
                {
                        // header
                        writer.Write((byte)BinLogTypes.TcpConnection);

                        // id as (length, UTF-8)
                        byte[] idBuf = Encoding.UTF8.GetBytes(id);
                        writer.Write((UInt32)idBuf.Length);
                        writer.Write(idBuf);

                        // timestamps as ticks
                        writer.Write((UInt64)startTimestamp.Ticks);
                        writer.Write((UInt64)localEndTimestamp.Ticks);
                        writer.Write((UInt64)remoteEndTimestamp.Ticks);

                        // end points as (family as int, ip length, ip as bytes, port)
                        WriteEndPoint(writer, localPoint);
                        WriteEndPoint(writer, remotePoint);

                        // states as byte
                        writer.Write((byte)localState);
                        writer.Write((byte)remoteState);

                        // each tcp message as (direction as byte, timestamp in ticks, length, content)
                        foreach(TcpMessage message in messages) {
                                writer.Write((byte)message.Direction);
                                writer.Write((UInt64)message.Timestamp.Ticks);
                                writer.Write((UInt32)message.Length);
                                writer.Write(message.Bytes, 0, message.Length);
                        }

                        // end of stream marker
                        writer.Write((byte)TcpMessageDirection.None);
                }

                private static IPEndPoint ReadEndPoint(BinaryReader reader)
                {
                        // end point as (family as int, ip length, ip as bytes, port)
                        AddressFamily fam           = (AddressFamily)reader.ReadInt32();
                        int           addressLength = reader.ReadInt32();
                        byte[]        addressBytes  = reader.ReadBytes(addressLength);
                        int           port          = reader.ReadInt32();
                        IPAddress     address       = new IPAddress(addressBytes);

                        return new IPEndPoint(address, port);
                }

                internal static TcpConnection ReadBinLog(BinaryReader reader)
                {
                        // header
                        BinLogTypes recordType = (BinLogTypes)reader.ReadByte();

                        if(recordType == BinLogTypes.None)
                                return null;
                        else if(recordType == BinLogTypes.TcpConnection) {
                        }
                        else
                                throw new Exception("Wrong data type");

                        // id as (length, UTF-8)
                        int    idLength = (int)reader.ReadUInt32();
                        string id       = Encoding.UTF8.GetString(reader.ReadBytes(idLength));

                        TcpConnection tcp = new TcpConnection(id);

                        // timestamps as ticks
                        tcp.startTimestamp     = new DateTime((long)reader.ReadUInt64());
                        tcp.localEndTimestamp  = new DateTime((long)reader.ReadUInt64());
                        tcp.remoteEndTimestamp = new DateTime((long)reader.ReadUInt64());

                        // end points as (family as int, ip length, ip as bytes, port)
                        tcp.localPoint  = ReadEndPoint(reader);
                        tcp.remotePoint = ReadEndPoint(reader);

                        // states as byte - read but ignore
                        reader.ReadByte();
                        reader.ReadByte();

                        // each tcp message as (direction as byte, timestamp in ticks, length, content)
                        tcp.localState  = SocketState.Closed;
                        tcp.remoteState = SocketState.Closed;
                        for(;;) {
                                TcpMessageDirection direction = (TcpMessageDirection)reader.ReadByte();
                                if(direction == TcpMessageDirection.None) break; // end of stream marker

                                DateTime timestamp = new DateTime((long)reader.ReadUInt64());
                                int      bufLength = (int)reader.ReadUInt32();
                                byte[]   buf       = reader.ReadBytes(bufLength);

                                TcpMessage msg = tcp.Append(direction, buf);
                                msg.SetTimestamp(timestamp);
                        }
                        tcp.OnUpdate(new TcpEventArgs());

                        return tcp;
                }

                protected class SocketWorker
                {
                        private enum SendCommandType
                        {
                                Send,
                                Shutdown,
                                Reset
                        }

                        private class SendCommand
                        {
                                public byte[]          buffer  = null;
                                public int             length  = 0;
                                public SendCommandType cmdType = SendCommandType.Send;
                        }

                        private static int BUF_SIZE = 2048;

                        private TcpConnection    tcp;
                        private Socket           localSocket;
                        private bool             localSocketOpen             = false;
                        private Socket           remoteSocket;
                        private bool             remoteSocketOpen            = false;
                        private byte[]           localDataBuffer;
                        private byte[]           remoteDataBuffer;
                        private AsyncCallback    receiveLocalMethod;
                        private AsyncCallback    receiveRemoteMethod;
                        private Queue            localSendQueue              = new Queue();
                        private Queue            remoteSendQueue             = new Queue();
                        private AutoResetEvent   localSendEvent              = new AutoResetEvent(false);
                        private AutoResetEvent   remoteSendEvent             = new AutoResetEvent(false);
                        private bool             localSocketSendShutdown     = false;
                        private bool             localSocketReceiveShutdown  = false;
                        private bool             remoteSocketSendShutdown    = false;
                        private bool             remoteSocketReceiveShutdown = false;
                        private ManualResetEvent remoteSocketEvent           = new ManualResetEvent(false);
                        private Thread           localSendThread;
                        private Thread           remoteSendThread;

                        public SocketWorker(IPAddress resendHost, int resendPort, Socket localSocket, TcpConnection tcp)
                        {
                                try
                                {
                                        tcp.SendLog(LogLevel.Debug, string.Format("Local socket: {0}:{1} <-> {2}:{3}",
                                                ((IPEndPoint)localSocket.LocalEndPoint).Address,
                                                ((IPEndPoint)localSocket.LocalEndPoint).Port,
                                                ((IPEndPoint)localSocket.RemoteEndPoint).Address,
                                                ((IPEndPoint)localSocket.RemoteEndPoint).Port));

                                        this.localSocket     = localSocket;
                                        this.localSocketOpen = true;
                                        this.tcp             = tcp;
                                        receiveLocalMethod   = new AsyncCallback(OnLocalReceived);
                                        receiveRemoteMethod  = new AsyncCallback(OnRemoteReceived);

                                        tcp.SetLocalPoint((IPEndPoint)localSocket.RemoteEndPoint);
                                        this.localSocket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.NoDelay, 1);

                                        localSendThread = new Thread(new ThreadStart(LocalSendProc));
                                        ThreadDebugger.Add(localSendThread, "LocalSendProc");
                                        localSendThread.Name = "SocketWorker.LocalSendProc";
                                        localSendThread.Start();

                                        ContinueLocalReceive();

                                        if(resendHost == null)
                                        {
                                                remoteSocket = null;
                                        }
                                        else
                                        {
                                                tcp.SetRemoteState(SocketState.Connecting);

                                                IPEndPoint point = new IPEndPoint(resendHost, resendPort);
                                                remoteSocket = new Socket(point.AddressFamily, SocketType.Stream, ProtocolType.Tcp);
                                                remoteSocket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.NoDelay, 1);
                                                remoteSocket.Connect(point);
                                                remoteSocketOpen = true;
                                                tcp.SetRemoteState(SocketState.Connected);
                                                tcp.SetRemotePoint((IPEndPoint)remoteSocket.RemoteEndPoint);

                                                remoteSendThread = new Thread(new ThreadStart(RemoteSendProc));
                                                ThreadDebugger.Add(remoteSendThread, "RemoteSendProc");
                                                remoteSendThread.Name = "SocketWorker.RemoteSendProc";
                                                remoteSendThread.Start();

                                                ContinueRemoteReceive();
                                                remoteSocketEvent.Set(); // remote socket ready to send data
                                                tcp.SendLog(LogLevel.Info, "Connected to server " + tcp.RemotePoint.ToString());

                                                tcp.SendLog(LogLevel.Debug, string.Format("Remote socket: {0}:{1} <-> {2}:{3}",
                                                        ((IPEndPoint)remoteSocket.LocalEndPoint).Address,
                                                        ((IPEndPoint)remoteSocket.LocalEndPoint).Port,
                                                        ((IPEndPoint)remoteSocket.RemoteEndPoint).Address,
                                                        ((IPEndPoint)remoteSocket.RemoteEndPoint).Port));
                                        }
                                }
                                catch(Exception ex)
                                {
                                        tcp.SendLog(LogLevel.Warning, ex);
                                        Cancel();
                                }
                        }

                        private void ContinueLocalReceive()
                        {
                                try
                                {
                                        localDataBuffer = new byte[BUF_SIZE];
                                        localSocket.BeginReceive(localDataBuffer, 0, BUF_SIZE, SocketFlags.None, receiveLocalMethod, this);
                                }
                                catch(ObjectDisposedException ex) // the socket is closed
                                {
                                        tcp.SendLog(LogLevel.Info, ex);
                                        Cancel();
                                }
                                catch(Exception ex)
                                {
                                        tcp.SendLog(LogLevel.Warning, ex);
                                        Cancel();
                                }
                        }

                        private void ContinueRemoteReceive()
                        {
                                try
                                {
                                        remoteDataBuffer = new byte[BUF_SIZE];
                                        remoteSocket.BeginReceive(remoteDataBuffer, 0, BUF_SIZE, SocketFlags.None, receiveRemoteMethod, this);
                                }
                                catch(ObjectDisposedException ex) // the socket is closed
                                {
                                        tcp.SendLog(LogLevel.Info, ex);
                                        Cancel();
                                }
                                catch(Exception ex)
                                {
                                        tcp.SendLog(LogLevel.Warning, ex);
                                        Cancel();
                                }
                        }

                        private void CheckLocalSocket()
                        {
                                lock(localSocket)
                                {
                                        try
                                        {
                                                if(localSocketReceiveShutdown && localSocketSendShutdown)
                                                {
                                                        if(localSocket.Connected) localSocket.Close();
                                                        tcp.SetLocalState(SocketState.Closed);
                                                }
                                        }
                                        catch(Exception ex) // in any case we want to close the socket
                                        {
                                                tcp.SendLog(LogLevel.Warning, ex);
                                        }
                                }
                        }

                        private void CheckRemoteSocket()
                        {
                                lock(remoteSocket)
                                {
                                        try
                                        {
                                                if(remoteSocketReceiveShutdown && remoteSocketSendShutdown)
                                                {
                                                        if(remoteSocket.Connected) remoteSocket.Close();
                                                        tcp.SetRemoteState(SocketState.Closed);
                                                }
                                        }
                                        catch(Exception ex) // in any case we want to close the socket
                                        {
                                                tcp.SendLog(LogLevel.Warning, ex);
                                        }
                                }
                        }

                        private void OnLocalReceived(IAsyncResult asyn)
                        {
                                if(!localSocketOpen) return; // we have already closed the socket

                                try
                                {
                                        int  bytesReceived = 0;
                                        bool reset         = false;

                                        try
                                        {
                                                bytesReceived = localSocket.EndReceive(asyn);
                                        }
                                        catch(ObjectDisposedException)
                                        {
                                                reset = true;
                                        }
                                        catch(SocketException ex)
                                        {
                                                if(ex.ErrorCode == 10054)
                                                        reset = true;
                                                else
                                                        throw ex;
                                        }

                                        if(reset)
                                        {
                                                tcp.SendLog(LogLevel.Info, "Got reset from local end");

                                                lock(localSocket)
                                                {
                                                        if(localSocket.Connected) localSocket.Close();
                                                        tcp.SetLocalState(SocketState.Closed);
                                                }

                                                SendCommand cmd = new SendCommand();
                                                cmd.cmdType = SendCommandType.Reset;
                                                lock(localSendQueue)
                                                {
                                                        localSendQueue.Enqueue(cmd);
                                                        localSendEvent.Set();
                                                }
                                        }
                                        else if(bytesReceived <= 0)
                                        {
                                                tcp.SendLog(LogLevel.Info, "Got showdown from local end");

                                                localSocket.Shutdown(SocketShutdown.Receive);
                                                tcp.SetLocalState(SocketState.ShutdownReceived);
                                                localSocketReceiveShutdown = true;
                                                CheckLocalSocket();

                                                SendCommand cmd = new SendCommand();
                                                cmd.cmdType = SendCommandType.Shutdown;
                                                lock(localSendQueue)
                                                {
                                                        localSendQueue.Enqueue(cmd);
                                                        localSendEvent.Set();
                                                }
                                        }
                                        else
                                        {
                                                tcp.SendLog(LogLevel.Debug, string.Format("Local received {0} bytes", bytesReceived));

                                                SendCommand cmd = new SendCommand();
                                                cmd.buffer = localDataBuffer;
                                                cmd.length = bytesReceived;
                                                lock(localSendQueue)
                                                {
                                                        localSendQueue.Enqueue(cmd);
                                                        localSendEvent.Set();
                                                }
                                                ContinueLocalReceive();
                                        }
                                }
                                catch(Exception ex)
                                {
                                        tcp.SendLog(LogLevel.Warning, ex);
                                        Cancel();
                                }
                        }

                        private void LocalSendProc()
                        {
                                try
                                {
                                        while(true)
                                        {
                                                SendCommand cmd;

                                                if(localSendQueue.Count == 0)
                                                {
                                                        localSendEvent.WaitOne();
                                                }

                                                lock(localSendQueue)
                                                {
                                                        localSendEvent.Reset();
                                                        cmd = (SendCommand)localSendQueue.Dequeue();
                                                }

                                                if(cmd.cmdType == SendCommandType.Reset) // reset marker
                                                {
                                                        if(remoteSocket == null || !remoteSocket.Connected) remoteSocketEvent.WaitOne();
                                                        tcp.SendLog(LogLevel.Debug, string.Format("Send reset to remote end"));
                                                        lock(remoteSocket)
                                                        {
                                                                if(!remoteSocket.Connected) remoteSocket.Close();
                                                                tcp.SetRemoteState(SocketState.Closed);
                                                        }

                                                        break; // no more send allowed
                                                }
                                                else if(cmd.cmdType == SendCommandType.Shutdown) // shutdown marker
                                                {
                                                        if(remoteSocket == null || !remoteSocket.Connected) remoteSocketEvent.WaitOne();
                                                        tcp.SendLog(LogLevel.Debug, string.Format("Send shutdown to remote end"));
                                                        remoteSocket.Shutdown(SocketShutdown.Send);
                                                        tcp.SetRemoteState(SocketState.ShutdownSend);
                                                        remoteSocketSendShutdown = true;
                                                        CheckRemoteSocket();

                                                        break; // no more send allowed
                                                }
                                                else
                                                {
                                                        // store received bytes
                                                        tcp.Append(TcpMessageDirection.Local, cmd.buffer, cmd.length);

                                                        // forward it
                                                        if(remoteSocket == null || !remoteSocket.Connected) remoteSocketEvent.WaitOne();
                                                        tcp.SendLog(LogLevel.Debug, string.Format("Send {0} bytes to remote end", cmd.length));
                                                        remoteSocket.Send(cmd.buffer, cmd.length, SocketFlags.None);
                                                }
                                        }
                                }
                                catch(ThreadAbortException)
                                {
                                }
                                catch(Exception ex)
                                {
                                        tcp.SendLog(LogLevel.Warning, ex);
                                        Cancel();
                                }
                        }

                        private void OnRemoteReceived(IAsyncResult asyn)
                        {
                                if(!remoteSocketOpen) return; // we have already closed the socket

                                try
                                {
                                        int  bytesReceived = 0;
                                        bool reset         = false;

                                        try
                                        {
                                                bytesReceived = remoteSocket.EndReceive(asyn);
                                        }
                                        catch(ObjectDisposedException)
                                        {
                                                reset = true;
                                        }
                                        catch(SocketException ex)
                                        {
                                                if(ex.ErrorCode == 10054)
                                                        reset = true;
                                                else
                                                        throw ex;
                                        }

                                        if(reset)
                                        {
                                                tcp.SendLog(LogLevel.Info, "Got reset from remote end");

                                                lock(remoteSocket)
                                                {
                                                        if(remoteSocket.Connected) remoteSocket.Close();
                                                        tcp.SetRemoteState(SocketState.Closed);
                                                }

                                                SendCommand cmd = new SendCommand();
                                                cmd.cmdType = SendCommandType.Reset;
                                                lock(remoteSendQueue)
                                                {
                                                        remoteSendQueue.Enqueue(cmd);
                                                        remoteSendEvent.Set();
                                                }
                                        }
                                        else if(bytesReceived <= 0)
                                        {
                                                tcp.SendLog(LogLevel.Info, "Got showdown from remote end");

                                                remoteSocket.Shutdown(SocketShutdown.Receive);
                                                tcp.SetRemoteState(SocketState.ShutdownReceived);
                                                remoteSocketReceiveShutdown = true;
                                                CheckRemoteSocket();

                                                SendCommand cmd = new SendCommand();
                                                cmd.cmdType = SendCommandType.Shutdown;
                                                lock(remoteSendQueue)
                                                {
                                                        remoteSendQueue.Enqueue(cmd);
                                                        remoteSendEvent.Set();
                                                }
                                        }
                                        else
                                        {
                                                tcp.SendLog(LogLevel.Debug, string.Format("Remote received {0} bytes", bytesReceived));

                                                SendCommand cmd = new SendCommand();
                                                cmd.buffer = remoteDataBuffer;
                                                cmd.length = bytesReceived;
                                                lock(remoteSendQueue)
                                                {
                                                        remoteSendQueue.Enqueue(cmd);
                                                        remoteSendEvent.Set();
                                                }
                                                ContinueRemoteReceive();
                                        }
                                }
                                catch(Exception ex)
                                {
                                        tcp.SendLog(LogLevel.Warning, ex);
                                        Cancel();
                                }
                        }

                        private void RemoteSendProc()
                        {
                                try
                                {
                                        while(true)
                                        {
                                                SendCommand cmd;

                                                if(remoteSendQueue.Count == 0)
                                                {
                                                        remoteSendEvent.WaitOne();
                                                }

                                                lock(remoteSendQueue)
                                                {
                                                        remoteSendEvent.Reset();
                                                        cmd = (SendCommand)remoteSendQueue.Dequeue();
                                                }

                                                if(cmd.cmdType == SendCommandType.Reset) // reset marker
                                                {
                                                        tcp.SendLog(LogLevel.Debug, string.Format("Send reset to local end"));
                                                        lock(localSocket)
                                                        {
                                                                if(localSocket.Connected) localSocket.Close();
                                                                tcp.SetLocalState(SocketState.Closed);
                                                        }

                                                        break; // no more send allowed
                                                }
                                                else if(cmd.cmdType == SendCommandType.Shutdown) // shutdown marker
                                                {
                                                        tcp.SendLog(LogLevel.Debug, string.Format("Send shutdown to local end"));
                                                        localSocket.Shutdown(SocketShutdown.Send);
                                                        tcp.SetLocalState(SocketState.ShutdownSend);
                                                        localSocketSendShutdown = true;
                                                        CheckLocalSocket();

                                                        break; // no more send allowed
                                                }
                                                else
                                                {
                                                        // store received bytes
                                                        tcp.Append(TcpMessageDirection.Remote, cmd.buffer, cmd.length);

                                                        // forward it
                                                        tcp.SendLog(LogLevel.Debug, string.Format("Send {0} bytes to local end", cmd.length));
                                                        localSocket.Send(cmd.buffer, cmd.length, SocketFlags.None);
                                                }
                                        }
                                }
                                catch(ThreadAbortException)
                                {
                                }
                                catch(Exception ex)
                                {
                                        tcp.SendLog(LogLevel.Warning, ex);
                                        Cancel();
                                }
                        }

                        public void Cancel()
                        {
                                tcp.SendLog(LogLevel.Important, "Connection canceled");

                                try
                                {
                                        if(localSendThread != null  && localSendThread.IsAlive)  localSendThread.Abort();
                                        if(remoteSendThread != null && remoteSendThread.IsAlive) remoteSendThread.Abort();

                                        // close sockets
                                        try
                                        {
                                                if(localSocket != null)
                                                {
                                                        lock(localSocket)
                                                        {
                                                                if(localSocket.Connected) localSocket.Close();
                                                        }
                                                }
                                        }
                                        catch(Exception ex) // in any case we want to close the socket
                                        {
                                                tcp.SendLog(LogLevel.Warning, ex);
                                        }
                                        tcp.SetLocalState(SocketState.Closed);
                                        localSocketOpen = false;

                                        try
                                        {
                                                if(remoteSocket != null)
                                                {
                                                        lock(remoteSocket)
                                                        {
                                                                if(remoteSocket.Connected) remoteSocket.Close();
                                                        }
                                                }
                                        }
                                        catch(Exception ex) // in any case we want to close the socket
                                        {
                                                tcp.SendLog(LogLevel.Warning, ex);
                                        }
                                        tcp.SetRemoteState(SocketState.Closed);
                                        remoteSocketOpen = false;

                                        // return
                                        tcp.OnClose(new TcpEventArgs());
                                }
                                catch(Exception ex)
                                {
                                        tcp.SendLog(LogLevel.Warning, ex);
                                }
                        }

                        public void WaitCancel()
                        {
                                // wait for the threads
                                localSendThread.Join();
                                remoteSendThread.Join();
                        }
                }
        }

        internal class HttpParser
        {
                private enum HttpCharType
                {
                        None,
                        Control,
                        Digit,
                        UpAlpha,
                        LoAlpha,
                        NonChar,
                        Separator,
                        CrLf,
                }

                private static HttpCharType[] charTypes  = null;
                private static bool[]         tokenChars = null;
                private static char[]         charValues = {
                         '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0',
                         '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0', '\0',
                         ' ',  '!',  '"',  '#',  '$',  '%',  '&',  '\'', '(',  ')',  '*',  '+',  ',',  '-',  '.',  '/',
                         '0',  '1',  '2',  '3',  '4',  '5',  '6',  '7',  '8',  '9',  ':',  ';',  '<',  '=',  '>',  '?',
                         '@',  'A',  'B',  'C',  'D',  'E',  'F',  'G',  'H',  'I',  'J',  'K',  'L',  'M',  'N',  'O',
                         'P',  'Q',  'R',  'S',  'T',  'U',  'V',  'W',  'X',  'Y',  'Z',  '[',  '\\', ']',  '^',  '_',
                         '`',  'a',  'b',  'c',  'd',  'e',  'f',  'g',  'h',  'i',  'j',  'k',  'l',  'm',  'n',  'o',
                         'p',  'q',  'r',  's',  't',  'u',  'v',  'w',  'x',  'y',  'z',  '{',  '|',  '}',  '~',  '\0'
                 };

                private static void InitTables()
                {
                        if(charTypes != null) return;

                        // main table
                        charTypes = new HttpCharType[256];

                        for(int i = 0; i < charTypes.Length; i++) charTypes[i] = HttpCharType.None;

                        for(int i = 0; i <= 31; i++) charTypes[i] = HttpCharType.Control;
                        charTypes[127] = HttpCharType.Control;   // <del>

                        for(int i = 48; i <= 57; i++) charTypes[i] = HttpCharType.Digit;

                        for(int i = 65; i <=  90; i++) charTypes[i] = HttpCharType.UpAlpha;
                        for(int i = 97; i <= 122; i++) charTypes[i] = HttpCharType.LoAlpha;

                        for(int i = 128; i < charTypes.Length; i++) charTypes[i] = HttpCharType.NonChar;

                        charTypes[ 40] = HttpCharType.Separator;   // (
                        charTypes[ 41] = HttpCharType.Separator;   // )
                        charTypes[ 60] = HttpCharType.Separator;   // <
                        charTypes[ 62] = HttpCharType.Separator;   // >
                        charTypes[ 64] = HttpCharType.Separator;   // @
                        charTypes[ 44] = HttpCharType.Separator;   // ,
                        charTypes[ 59] = HttpCharType.Separator;   // ;
                        charTypes[ 58] = HttpCharType.Separator;   // :
                        charTypes[ 92] = HttpCharType.Separator;   // \
                        charTypes[ 34] = HttpCharType.Separator;   // "
                        charTypes[ 47] = HttpCharType.Separator;   // /
                        charTypes[ 91] = HttpCharType.Separator;   // [
                        charTypes[ 93] = HttpCharType.Separator;   // ]
                        charTypes[ 63] = HttpCharType.Separator;   // ?
                        charTypes[ 61] = HttpCharType.Separator;   // =
                        charTypes[123] = HttpCharType.Separator;   // {
                        charTypes[125] = HttpCharType.Separator;   // }
                        charTypes[ 32] = HttpCharType.Separator;   // <space>
                        charTypes[  9] = HttpCharType.Separator;   // <tab>

                        charTypes[ 13] = HttpCharType.CrLf;        // <CR>
                        charTypes[ 10] = HttpCharType.CrLf;        // <LF>

                        // token table
                        tokenChars = new bool[256];
                        for(int i = 0; i < tokenChars.Length; i++)
                        {
                                tokenChars[i] = !(charTypes[i] == HttpCharType.NonChar
                                        || charTypes[i] == HttpCharType.Control || charTypes[i] == HttpCharType.Separator
                                        || charTypes[i] == HttpCharType.CrLf);
                        }
                }

                private class ParsePosition
                {
                        private TcpConnection  messages;
                        private IEnumerator    messagesEnum;
                        private TcpMessage     tcp    = null;
                        private int            tcpPos;
                        private int            tcpLen;
                        private bool           tcpEnd = false;
                        private AutoResetEvent newMessageEvent;
                        private AutoResetEvent nextMessageEvent;

                        public ParsePosition(TcpConnection messages, AutoResetEvent nextMessageEvent)
                        {
                                this.messages         = messages;
                                this.messagesEnum     = messages.Messages.GetEnumerator();
                                this.newMessageEvent  = new AutoResetEvent(false);
                                this.nextMessageEvent = nextMessageEvent;
                        }

                        public AutoResetEvent NewMessageEvent
                        {
                                get { return newMessageEvent; }
                        }

                        public bool IsEnd
                        {
                                get { return tcpEnd; }
                        }

                        public TcpMessage CurrentMessage
                        {
                                get { return tcp; }
                        }

                        private bool MoveNext()
                        {
                                for(bool moved = false; !moved; )
                                {
                                        lock(messages)
                                        {
                                                newMessageEvent.Reset();
                                                moved = messagesEnum.MoveNext();

                                                if(!moved && messages.LocalState == SocketState.Closed && messages.RemoteState == SocketState.Closed)
                                                        return false;
                                        }

                                        if(moved) break;

                                        if(!newMessageEvent.WaitOne())
                                                throw new Exception("Cannot get next TCP message");

                                        lock(messages)
                                        {
                                                if(messages.LocalState == SocketState.Closed && messages.RemoteState == SocketState.Closed)
                                                        return false;

                                                moved = messagesEnum.MoveNext();
                                        }
                                }

                                return true;
                        }

                        private void NextTcp()
                        {
                                if(tcpEnd) return;

                                TcpMessage newTcp = null;

                                do
                                {
                                        if(!MoveNext())
                                        {
                                                tcpEnd = true;
                                                break;
                                        }
                                        newTcp = (TcpMessage)messagesEnum.Current;
                                }
                                while(tcp != null && tcp.Direction != newTcp.Direction);
                                // FIXME correct? no hang?

                                if(!tcpEnd)
                                {
                                        tcp    = newTcp;
                                        tcpLen = tcp.Length;
                                        tcpPos = -1;
                                        if(nextMessageEvent != null) nextMessageEvent.Set();
                                }
                        }

                        public void Back()
                        {
                                tcpPos--; // FIXME double check if it's always possible to go to _one_ step back
                        }

                        public byte NextOctet()
                        {
                                tcpPos++;

                                if(tcp == null || tcpPos >= tcpLen)
                                {
                                        NextTcp();
                                        tcpPos++;
                                }
                                if(tcpEnd) return 0;

                                return tcp.Bytes[tcpPos];
                        }

                        public byte NextOctetAndBack()
                        {
                                byte b = NextOctet();
                                if(!tcpEnd) Back();

                                return b;
                        }

                        public void SetDirection(TcpMessageDirection direction)
                        {
                                do
                                {
                                        if(!MoveNext())
                                        {
                                                tcp    = null;
                                                tcpEnd = true;
                                                break;
                                        }
                                        tcp = (TcpMessage)messagesEnum.Current;
                                }
                                while(tcp.Direction != direction);

                                if(!tcpEnd)
                                {
                                        tcpLen = tcp.Length;
                                        tcpPos = -1;
                                }
                        }
                }

                private TcpConnection    messages;
                private AutoResetEvent   requestEvent     = new AutoResetEvent(false); // new request found
                private AutoResetEvent   nextMessageEvent = new AutoResetEvent(false); // request goes to next TCP message
                private LinkedList       https            = new LinkedList();
                private ParsePosition    requestPos;
                private ParsePosition    responsePos;
                private Thread           runThread;

                public static HttpParser Parse(TcpConnection messages)
                {
                        HttpParser parser = new HttpParser(messages);
                        parser.runThread = new Thread(new ThreadStart(parser.Run));
                        ThreadDebugger.Add(parser.runThread, "HttpParser.Run");
                        parser.runThread.Name = "HttpParser.Run";
                        parser.runThread.Start();

                        return parser;
                }

                public void NewMessageArived()
                {
                        requestPos.NewMessageEvent.Set();
                        responsePos.NewMessageEvent.Set();
                }

                public Thread RunThread
                {
                        get { return runThread; }
                }

                private HttpParser(TcpConnection messages)
                {
                        this.messages    = messages;
                        this.requestPos  = new ParsePosition(messages, nextMessageEvent);
                        this.responsePos = new ParsePosition(messages, null);
                        InitTables();
                }

                /// <summary>
                /// Try to recognize the stored TCP packets as sequence of HTTP messages (request-response)
                /// </summary>
                private void Run()
                {
                        Thread responseThread = null;

                        try
                        {
                                responseThread = new Thread(new ThreadStart(MatchResponses));
                                ThreadDebugger.Add(responseThread, "MatchResponses");
                                responseThread.Name = "HttpParser.MatchResponses";
                                responseThread.Start();

                                // find requests
                                while(!requestPos.IsEnd)
                                {
                                        HttpMessage http = new HttpMessage();
                                        lock(https)
                                        {
                                                https.Add(http);
                                                requestEvent.Set(); // new request available
                                        }

                                        messages.AddHttpMessage(http);
                                        SkipEmptyLines(requestPos);
                    http.Request.StartTimestamp = requestPos.CurrentMessage.Timestamp;

                                        ParseRequestLine(requestPos, http);
                                        http.UpdateHttpMessage();

                                        ParseHeaders(requestPos, http, http.Request);
                                        SetRequestProperties(http, http.Request);
                                        http.UpdateHttpMessage();

                                        ParseBody(requestPos, http, http.Request);
                                        if("text" == http.Request.ContentType && "xml" == http.Request.ContentSubtype)
                                        {
                                                http.Request.Xml = new XmlMessage(http.Request.Text);
                                        }
                                        http.UpdateHttpMessage();

                                        http.Request.Complete = true;
                                        http.UpdateHttpMessage();

                                        SkipEmptyLines(requestPos);
                                }

                                responseThread.Join();
                        }
                        catch(Exception ex)
                        {
                                Console.WriteLine(ex.Message + " (" + ex.GetType().Name + ")\n" + ex.StackTrace);
                                if(responseThread != null) responseThread.Abort();
                        }
                }

                private void MatchResponses()
                {
                        try
                        {
                                IEnumerator httpEnum = https.GetEnumerator();

                                if(!nextMessageEvent.WaitOne()) throw new Exception("Cannot get first message of request");

                                responsePos.SetDirection(requestPos.CurrentMessage.Direction == TcpMessageDirection.Local
                                        ? TcpMessageDirection.Remote : TcpMessageDirection.Local);

                                while(!responsePos.IsEnd)
                                {
                                        bool moved;

                                        lock(https)
                                        {
                                                requestEvent.Reset();
                                                moved = httpEnum.MoveNext();
                                        }

                                        if(!moved)
                                        {
                                                if(!requestEvent.WaitOne())
                                                        throw new Exception("Cannot get next request");

                                                lock(https)
                                                {
                                                        if(!httpEnum.MoveNext())
                                                                throw new Exception("Tried to find response but no HTTP message available");
                                                }
                                        }

                                        HttpMessage http = (HttpMessage)httpEnum.Current;

                                        ParseResponse(responsePos, http);

                                        bool continueResponse = false;
                                        lock(http)
                                        {
                                                if(http.Response.StatusCode == 100) // "100 (Continue)" response
                                                {
                                                        continueResponse = true;
                                                        http.GotContinueResponse();
                                                }
                                        }
                                        if(continueResponse) ParseResponse(responsePos, http); // once again

                                        http.Response.Complete = true;
                                        http.UpdateHttpMessage();
                                        responsePos.NextOctetAndBack(); // go forward to see end of connection
                                }
                        }
                        catch(Exception ex)
                        {
                                Console.WriteLine(ex.Message + " (" + ex.GetType().Name + ")\n" + ex.StackTrace);
                        }
                }

                private string GetToken(ParsePosition pos, int limit)
                {
                        StringBuilder res = new StringBuilder(100);
                        int           len = 0;

                        for(byte b = pos.NextOctet(); !pos.IsEnd && tokenChars[b]; b = pos.NextOctet())
                        {
                                res.Append(charValues[b]);
                                if(limit > 0 && limit < ++len) return null; // length limit
                        }
                        pos.Back();

                        return res.ToString();
                }

                private string GetUntilSpace(ParsePosition pos, int limit)
                {
                        StringBuilder res = new StringBuilder(1024);
                        int           len = 0;

                        for(byte b = pos.NextOctet(); !pos.IsEnd && b != 32 && b != 13 && b != 10; b = pos.NextOctet())
                        {                                     //     <space>    <cr>       <lf>
                                res.Append(charValues[b]);
                                if(limit > 0 && limit < ++len) return null; // length limit
                        }
                        pos.Back();

                        return res.ToString();
                }

                private string GetUntilEoL(ParsePosition pos, int limit)
                {
                        StringBuilder res = new StringBuilder(1024);
                        int           len = 0;

                        for(byte b = pos.NextOctet(); !pos.IsEnd && b != 13; b = pos.NextOctet())
                        {                                     //     <cr>
                                res.Append(charValues[b]);
                                if(limit > 0 && limit < ++len) return null; // length limit
                        }
                        pos.Back();

                        return res.ToString();
                }

                private void ExpectSpace(ParsePosition pos)
                {
                        if(pos.IsEnd || pos.NextOctet() != 32)
                                throw new HttpParseException("Space expected");
                }

                private void ExpectCRLF(ParsePosition pos)
                {
                        if(pos.IsEnd || pos.NextOctet() != 13)
                                throw new HttpParseException("Carriage return expected");
                        if(pos.IsEnd || pos.NextOctet() != 10)
                                throw new HttpParseException("Linefeed expected");
                }

                private void SkipEmptyLines(ParsePosition pos)
                {
                        while(pos.NextOctetAndBack() == 13)
                        {
                                ExpectCRLF(pos);
                        }
                }

                private void ParseRequestLine(ParsePosition pos, HttpMessage http)
                {
                        // method
                        http.Request.Method = GetToken(pos, 1024);
                        if(http.Request.Method == null || http.Request.Method.Length == 0)
                                throw new HttpParseException("Request method name expected");
                        ExpectSpace(pos);

                        // URI
                        http.Request.Uri = GetUntilSpace(pos, 1024);
                        if(http.Request.Uri == null || http.Request.Uri.Length == 0)
                                throw new HttpParseException("Request URI expected");

                        if(pos.IsEnd)
                                throw new HttpParseException("Unexpected end of message");

                        // EoL or version
                        byte b = pos.NextOctet();
                        if(b == 13)
                        {
                                pos.Back();
                                ExpectCRLF(pos);
                                http.Request.Version = HttpVersion.V0_9;
                        }
                        else if(b != 32)
                        {
                                throw new HttpParseException("HTTP version expected");
                        }

                        // check version
                        string versionStr = GetUntilEoL(pos, 20);
                        if(pos.IsEnd || versionStr == null || versionStr.Length == 0)
                                throw new HttpParseException("HTTP version expected");

                        if(versionStr == "HTTP/1.0")
                        {
                                http.Request.Version = HttpVersion.V1_0;
                        }
                        else if(versionStr == "HTTP/1.1")
                        {
                                http.Request.Version = HttpVersion.V1_1;
                        }
                        else
                        {
                                throw new HttpParseException("Unknown HTTP version: " + versionStr);
                        }

                        ExpectCRLF(pos);
                }

                private void ParseHeaders(ParsePosition pos, HttpMessage http, HttpHalfMessage half)
                {
                        if(pos.IsEnd) return;  // end of TCP messages

                        while(true)
                        {
                                if(pos.IsEnd)
                                        throw new HttpParseException("Unexpected end of message");

                                if(pos.NextOctetAndBack() == 13)
                                {
                                        ExpectCRLF(pos);
                                        return; // end of header, move to body
                                }
                                if(pos.IsEnd) return;  // end of TCP messages

                                string name = GetToken(pos, 0);
                                if(name == null || name.Length == 0)
                                        throw new HttpParseException("Header name expected");

                                if(pos.IsEnd || pos.NextOctet() != 58)   // :
                                        throw new HttpParseException("Header value expected");

                                string s = TrimHeaderValue(GetUntilEoL(pos, 0));

                                ExpectCRLF(pos);

                                half.AddHeader(name, s);
                        }
                }

                enum HeaderValueState
                {
                        Space,
                        Token,
                        Quoted
                }

                private string TrimHeaderValue(string s)
                {
                        if(s == null) return null;

                        HeaderValueState state = HeaderValueState.Space;
                        StringBuilder    buf   = new StringBuilder();

                        for(int i = 0, l = s.Length; i < l; i++)
                        {
                                char c = s[i];
                                switch(state)
                                {
                                        case HeaderValueState.Space:
                                                if(c != ' ' && c != '\t')
                                                {
                                                        if(c == '"')
                                                        {
                                                                if(buf.Length > 0) buf.Append(' ');
                                                                buf.Append(c);
                                                                state = HeaderValueState.Quoted;
                                                        }
                                                        else
                                                        {
                                                                if(buf.Length > 0) buf.Append(' ');
                                                                buf.Append(c);
                                                                state = HeaderValueState.Token;
                                                        }
                                                }
                                                break;

                                        case HeaderValueState.Token:
                                                if(c == ' ' || c == '\t')
                                                {
                                                        state = HeaderValueState.Space;
                                                }
                                                else if(c == '"')
                                                {
                                                        buf.Append(c);
                                                        state = HeaderValueState.Quoted;
                                                }
                                                else
                                                {
                                                        buf.Append(c);
                                                }
                                                break;

                                        case HeaderValueState.Quoted:
                                                if(c == '"')
                                                {
                                                        buf.Append(c);
                                                        i++;
                                                        if(i < l)
                                                        {
                                                                c = s[i];
                                                                if(c == ' ' || c == '\t')
                                                                {
                                                                        state = HeaderValueState.Space;
                                                                }
                                                                else if(c == '"')
                                                                {
                                                                        buf.Append(c);
                                                                        state = HeaderValueState.Quoted;
                                                                }
                                                                else
                                                                {
                                                                        buf.Append(c);
                                                                        state = HeaderValueState.Token;
                                                                }
                                                        }
                                                }
                                                else
                                                {
                                                        buf.Append(c);
                                                }
                                                break;
                                }
                        }

                        return buf.ToString();
                }

                private HttpTransferEncoding ParseTransferEncoding(string encoding)
                {
                        if(encoding == null) return HttpTransferEncoding.None;
                        encoding = encoding.ToLower();

                        if(encoding == "identity")
                                return HttpTransferEncoding.Identity;
                        else if(encoding == "chunked")
                                return HttpTransferEncoding.Chunked;
                        else if(encoding == "gzip")
                                return HttpTransferEncoding.Gzip;
                        else if(encoding == "compress")
                                return HttpTransferEncoding.Compress;
                        else if(encoding == "deflate")
                                return HttpTransferEncoding.Deflate;
                        else
                                return HttpTransferEncoding.Unknown;
                }

                private HttpContentEncoding ParseContentEncoding(string encoding)
                {
                        if(encoding == null) return HttpContentEncoding.None;
                        encoding = encoding.ToLower();

                        if(encoding == "identity")
                                return HttpContentEncoding.Identity;
                        else if(encoding == "gzip")
                                return HttpContentEncoding.Gzip;
                        else if(encoding == "compress")
                                return HttpContentEncoding.Compress;
                        else if(encoding == "deflate")
                                return HttpContentEncoding.Deflate;
                        else
                                return HttpContentEncoding.Unknown;
                }

                private void SetHttpProperties(HttpMessage http, HttpHalfMessage half)
                {
                        // length
                        HttpHeader contentLength = half.GetHeader("content-length");
                        if(contentLength != null)
                        {
                                half.Length = int.Parse(contentLength.Values[0]);
                        }

                        // transfer encoding
                        HttpHeader transferEncoding = half.GetHeader("transfer-encoding");
                        half.TransferEncoding = ParseTransferEncoding((transferEncoding == null) ? null : transferEncoding.Values[0]);
                        if(HasBody(http, half) && half.TransferEncoding == HttpTransferEncoding.None)
                                half.TransferEncoding = HttpTransferEncoding.Identity;

                        // content encoding
                        HttpHeader contentEncoding = half.GetHeader("content-encoding");
                        half.ContentEncoding = ParseContentEncoding((contentEncoding == null) ? null : contentEncoding.Values[0]);
                        if(HasBody(http, half) && half.ContentEncoding == HttpContentEncoding.None)
                                half.ContentEncoding = HttpContentEncoding.Identity;

                        // type & charset
                        HttpHeader contentType = half.GetHeader("content-type");
                        if(contentType != null)
                        {
                                Match match = Regex.Match(contentType.Values[0], @"^\s*([!#$%&'*+.0-9A-Z^_a-z|~-]+)/([!#$%&'*+.0-9A-Z^_a-z|~-]+)\s*($|;\s*(charset=""?(\S+)""?)?)");
                                if(match.Success)
                                {
                                        half.ContentType    = match.Groups[1].Captures[0].Value;
                                        half.ContentSubtype = match.Groups[2].Captures[0].Value;
                                        if(match.Groups.Count >= 6 && match.Groups[5].Captures.Count > 0)
                                                half.Charset = match.Groups[5].Captures[0].Value.Trim('"');
                                }
                        }
                }

                private void SetRequestProperties(HttpMessage http, HttpRequest request)
                {
                        SetHttpProperties(http, request);

                        // soap action
                        HttpHeader soapAction = request.GetHeader("soapaction");
                        if(soapAction != null)
                        {
                                request.SoapAction = soapAction.Values[0];
                        }
                }

                // RFC 2616: 4.3
                private bool HasBody(HttpMessage http, HttpHalfMessage half)
                {
                        if(half.IsRequest)
                        {
                                return (http.Request.Length > 0) 
                                        || (http.Request.TransferEncoding != HttpTransferEncoding.None);
                        }
                        else 
                        {
                                if(http.Request.Method == "HEAD") return false;
                                if(http.Response.StatusCode <  200) return false;
                                if(http.Response.StatusCode == 204) return false;
                                if(http.Response.StatusCode == 304) return false;

                                return true;
                        }
                }

                private void ParseBody(ParsePosition pos, HttpMessage http, HttpHalfMessage half)
                {
                        if(!HasBody(http, half)) return;

                        // FIXME parse and save on-the-fly, dont wait util end of message

                        byte[]               bin              = new byte[8*1024];
                        int                  len              = 0;   // current bin biffer length
                        int                  limit            = half.Length;
                        int                  chunkLen         = -1;  // current length of current chunk
                        int                  chunkLimit       = -1;
                        HttpTransferEncoding transferEncoding = half.TransferEncoding;
                        HttpContentEncoding  contentEncoding  = half.ContentEncoding;
                        string               contentType      = half.ContentType;
                        string               contentSubtype   = half.ContentSubtype;
                        string               charset          = half.Charset;

                        for(byte b = pos.NextOctet(); !pos.IsEnd; b = pos.NextOctet())
                        {
                                // RFC 2616: 3.6.1
                                if(transferEncoding == HttpTransferEncoding.Chunked && chunkLimit < 0)
                                {
                                        // FIXME recognize chunk-extension here

                                        // get chunk length
                                        pos.Back();
                                        string chunkLimitStr = GetUntilEoL(pos, 40);
                                        if(pos.IsEnd || chunkLimitStr == null || chunkLimitStr.Length == 0)
                                                throw new HttpParseException("Chunk length expected");

                                        try
                                        {
                                                chunkLimit = Convert.ToInt32(chunkLimitStr, 16);
                                        }
                                        catch(Exception)
                                        {
                                                throw new HttpParseException("Cannot parse chunk length");
                                        }

                                        ExpectCRLF(pos);

                                        if(chunkLimit == 0) { // the end marker
                                                ExpectCRLF(pos);
                                                break;
                                        }

                                        chunkLen = 0;
                                        b        = pos.NextOctet();
                                }

                                // grow array if full
                                if(len >= bin.Length)
                                {
                                        byte[] newBin = new byte[bin.Length*2];
                                        Array.Copy(bin, newBin, len);
                                        bin = newBin;
                                }

                                bin[len++] = b;
                                chunkLen++;
                                if(chunkLimit > 0)
                                {
                                        if(chunkLen >= chunkLimit)  // full chunk
                                        {
                                                ExpectCRLF(pos);
                                                chunkLimit = -1; // try to find length of next chunk on next iteration
                                        }
                                }
                                else 
                                {
                                        if(limit > 0 && len >= limit)  // full length
                                        {
                                                break;
                                        }
                                }
                        }
                        if(transferEncoding == HttpTransferEncoding.Chunked && chunkLimit > 0 && chunkLen != chunkLimit)
                        {
                                throw new HttpParseException("Incomplete chunk found");
                        }

                        // FIXME parse entity-headers for chunked encoding here

                        string text = null;
                        if(contentEncoding == HttpContentEncoding.Identity && contentType == "text")
                        {
                                try
                                {
                                        Encoding enc = Encoding.GetEncoding(charset == null ? (contentSubtype == "xml" ? "UTF-8" : "ASCII") : charset);
                                        text = enc.GetString(bin, 0, len);
                                }
                                catch(NotSupportedException)
                                {
                                        Console.WriteLine("Unsupported charset: " + charset);
                                }
                        }

                        half.Length = len;
                        half.Body   = bin;
                        half.Text   = text;
                }

                private void ParseResponse(ParsePosition pos, HttpMessage http)
                {
                        ParseResponseLine(pos, http);
                        http.Response.StartTimestamp = pos.CurrentMessage.Timestamp;
                        http.UpdateHttpMessage();

                        ParseHeaders(pos, http, http.Response);
                        SetHttpProperties(http, http.Response);
                        http.UpdateHttpMessage();

                        ParseBody(pos, http, http.Response);
                        if("text" == http.Response.ContentType && "xml" == http.Response.ContentSubtype)
                        {
                                http.Response.Xml = new XmlMessage(http.Response.Text);
                        }
                        http.UpdateHttpMessage();
                }

                private void ParseResponseLine(ParsePosition pos, HttpMessage http)
                {
                        // version
                        string versionStr = GetUntilSpace(pos, 20);
                        if(pos.IsEnd || versionStr == null || versionStr.Length == 0)
                                throw new HttpParseException("HTTP version expected");

                        if(versionStr == "HTTP/1.0")
                        {
                                http.Response.Version = HttpVersion.V1_0;
                        }
                        else if(versionStr == "HTTP/1.1")
                        {
                                http.Response.Version = HttpVersion.V1_1;
                        }
                        else
                        {
                                throw new HttpParseException("Unknown HTTP version: " + versionStr);
                        }
                        ExpectSpace(pos);

                        // status code
                        string code = GetToken(pos, 3);
                        if(code == null || code.Length != 3)
                                throw new HttpParseException("Status code expected");

                        try
                        {
                                int c = int.Parse(code);
                                if(c < 100 || c >= 1000) throw new HttpParseException("Status code expected");
                                http.Response.StatusCode = c;
                        }
                        catch(FormatException)
                        {
                                throw new HttpParseException("Status code expected");
                        }
                        ExpectSpace(pos);

                        // status message
                        http.Response.StatusMessage = GetUntilEoL(pos, 0);

                        if(pos.IsEnd)
                                throw new HttpParseException("Unexpected end of message");

                        ExpectCRLF(pos);
                }
        }

        public class HttpHeader
        {
                private string   name;
                private string[] headerValues;

                public string Name
                {
                        get { return name; }
                }

                public string[] Values
                {
                        get { return headerValues; }
                }

                internal HttpHeader()
                {
                }

                internal HttpHeader(string name, string headerValue)
                {
                        this.name = name;
                        AddValue(headerValue);
                }

                internal void AddValue(string value)
                {
                        if(headerValues == null)
                        {
                                headerValues = new string[1];
                        }
                        else
                        {
                                string[] newValues = new string[headerValues.Length + 1];
                                Array.Copy(headerValues, 0, newValues, 0, headerValues.Length);
                                headerValues = newValues;
                        }

                        headerValues[headerValues.Length-1] = value;
                }
        }

        public abstract class HttpHalfMessage
        {
                private   bool                 complete         = false;
                private   HttpVersion          version;
                protected LinkedList           headers          = new LinkedList();
                protected Hashtable            headersHash      = new Hashtable();
                private   int                  length           = -1; // -1 == unknown
                private   HttpTransferEncoding transferEncoding = HttpTransferEncoding.None;
                private   HttpContentEncoding  contentEncoding  = HttpContentEncoding.None;
                private   string               contentType;
                private   string               contentSubtype;
                private   string               charset;
                private   byte[]               body;
                private   string               text;
                private   XmlMessage           xml;
                private   DateTime             startTimestamp   = DateTime.MinValue;

                public abstract bool IsRequest
                {
                        get;
                }

                public bool Complete
                {
                        get { return complete; }
                        set { complete = value; }
                }

                public HttpVersion Version
                {
                        get { return version; }
                        set { version = value; }
                }

                public LinkedList Headers
                {
                        get { return headers; }
                }

                public int Length
                {
                        get { return length; }
                        set { length = value; }
                }

                public HttpTransferEncoding TransferEncoding
                {
                        get { return transferEncoding; }
                        set { transferEncoding = value; }
                }

                public HttpContentEncoding ContentEncoding
                {
                        get { return contentEncoding; }
                        set { contentEncoding = value; }
                }

                public string ContentType
                {
                        get { return contentType; }
                        set { contentType = value; }
                }

                public string ContentSubtype
                {
                        get { return contentSubtype; }
                        set { contentSubtype = value; }
                }

                public string Charset
                {
                        get { return charset; }
                        set { charset = value; }
                }

                public byte[] Body
                {
                        get { return body; }
                        set { body = value; }
                }

                public string Text
                {
                        get { return text; }
                        set { text = value; }
                }

                public XmlMessage Xml
                {
                        get { return xml; }
                        set { xml = value; }
                }

        public DateTime StartTimestamp
        {
                get { return startTimestamp; }
                set { startTimestamp = value; }
        }

                public void AddHeader(string name, string headerValue)
                {
                        HttpHeader header = (HttpHeader)headersHash[name];
                        if(header == null)
                        {
                                header = new HttpHeader(name, headerValue);
                                headers.Add(header);
                                headersHash.Add(name.ToLower(), header);
                        }
                        else
                        {
                                header.AddValue(headerValue);
                        }
                }

                public HttpHeader GetHeader(string name)
                {
                        return (HttpHeader)headersHash[name.ToLower()];
                }
        }

        public class HttpRequest : HttpHalfMessage
        {
                private string method;
                private string uri;
                private string soapAction;

                public override bool IsRequest
                {
                        get { return true; }
                }

                public string Method
                {
                        get { return method; }
                        set { method = value; }
                }

                public string Uri
                {
                        get { return uri; }
                        set { uri = value; }
                }

                public string SoapAction
                {
                        get { return soapAction; }
                        set { soapAction = value; }
                }
        }

        public class HttpResponse : HttpHalfMessage
        {
                private int    statusCode;
                private string statusMessage;

                public override bool IsRequest
                {
                        get { return false; }
                }

                public int StatusCode
                {
                        get { return statusCode; }
                        set { statusCode = value; }
                }

                public string StatusMessage
                {
                        get { return statusMessage; }
                        set { statusMessage = value; }
                }
        }

        public class HttpMessage
        {
                private HttpRequest  request          = new HttpRequest();
                private HttpResponse continueResponse = null;
                private HttpResponse response         = new HttpResponse();

                public HttpRequest Request
                {
                        get { return request; }
                }

                public HttpResponse ContinueResponse
                {
                        get { return continueResponse; }
                }

                public HttpResponse Response
                {
                        get { return response; }
                }

                public event TcpEventHandler Update;

                protected virtual void OnUpdate(TcpEventArgs e)
                {
                        if(Update != null)
                        {
                                Update(this, e);
                        }
                }

                internal void UpdateHttpMessage()
                {
                        OnUpdate(new TcpEventArgs());
                }

                internal void GotContinueResponse()
                {
                        continueResponse = response;
                        response = new HttpResponse();
                }
        }

        internal class HttpParseException : Exception
        {
                public HttpParseException() : base()
                {
                }

                public HttpParseException(string message) : base(message)
                {
                }

                public HttpParseException(System.Runtime.Serialization.SerializationInfo info,
                        System.Runtime.Serialization.StreamingContext context) : base(info, context)
                {
                }

                public HttpParseException(string message, Exception innerException) : base(message, innerException)
                {
                }
        }

        public class XmlMessage
        {
                private XmlDocument xml;
                private XmlException parseException;

                public XmlDocument Xml
                {
                        get { return xml; }
                }

                public XmlException ParseException
                {
                        get { return parseException; }
                }

                internal XmlMessage(string text)
                {
                        try
                        {
                                this.xml = new XmlDocument();
                                this.xml.LoadXml(text);
                        }
                        catch(XmlException ex)
                        {
                                parseException = ex;
                        }
                }
        }

        public class TcpMessage
        {
                private TcpMessageDirection direction = TcpMessageDirection.None;
                private byte[]              bytes;
                private int                 length = 0;
                private DateTime            timestamp;

                public TcpMessageDirection Direction
                {
                        get { return direction; }
                        set { direction = value; }
                }

                public int Length
                {
                        get { return length; }
                }

                public byte[] Bytes
                {
                        get
                        {
                                return bytes;
                        }
                        set
                        {
                                length = 0;
                                Append(value);
                        }
                }

                public DateTime Timestamp
                {
                        get { return timestamp; }
                }

                internal void SetTimestamp(DateTime timestamp)
                {
                        this.timestamp = timestamp;
                }

                internal TcpMessage()
                {
                        this.timestamp = DateTime.Now;
                        this.bytes  = new byte[1024];
                }

                internal TcpMessage(byte[] bytes, int length)
                {
                        this.timestamp = DateTime.Now;
                        this.bytes     = new byte[length];
                        this.length   = length;
                        Array.Copy(this.bytes, bytes, length);
                }

                internal TcpMessage Append(byte[] newBytes)
                {
                        if(newBytes == null) return this;

                        return Append(newBytes, newBytes.Length);
                }

                internal TcpMessage Append(byte[] newBytes, int length)
                {
                        if(newBytes == null) return this;

                        lock(this)
                        {
                                // grow array
                                if(this.length + length > bytes.Length)
                                {
                                        int newLength = bytes.Length;
                                        while(this.length + length > newLength) newLength *= 2;
                                        byte[] newArray = new byte[newLength];

                                        Array.Copy(bytes, newArray, this.length);
                                        bytes = newArray;
                                }

                                // store received bytes
                                Array.Copy(newBytes, 0, bytes, this.length, length);
                                this.length += length;

                                return this;
                        }
                }
        }

        public class ThreadDebugger
        {
                private static ArrayList threads = new ArrayList();

                internal static void Add(Thread thread, string comment)
                {
                        threads.Add(new ThreadItem(thread, comment));
                        Console.WriteLine("ThreadDebugger: thread added {0}", comment);
                }

                public static void PrintStatus()
                {
                        Console.WriteLine("=== ThreadDebugger Status Begin ======");
                        foreach(ThreadItem t in threads)
                        {
                                Console.WriteLine("{0} ({1}): {2}", t.thread.Name, t.comment, t.thread.IsAlive ? "alive" : "dead");
                        }
                        Console.WriteLine("=== ThreadDebugger Status End ========");
                }

                private class ThreadItem
                {
                        public Thread thread;
                        public string comment;

                        public ThreadItem(Thread thread, string comment)
                        {
                                this.thread  = thread;
                                this.comment = comment;
                        }
                }
        }
}