Random idea -> Data to Images

[Elio]

well there goes the multi uploader i was going to put at IC

[DeLeTeD]

lol elio it would only be good if it had multi download as well

[Hyperz]

Every descent download manager has that DeL . But tbh I don't think this is something you should worry about elio. If this would become a real problem for IC I'd be happy to show you how you can detect them and remove them via a cron job or something.

[Elio]

lol i was kidding but ok hypz

[Hyperz]

Lol ok then.

Here's what I have so far for the encoder part:

// DaToPic - Data To Picture Transcoder
// =============================================

#light

open System
open System.Drawing
open System.Drawing.Imaging
open System.IO
open System.Text
open System.Threading
open System.Threading.Tasks
open System.Windows.Forms

// =============================================
// =============================================

let MaxFileNameSize = 0xff // 255
let FileVersion = 0x01 // 1
let DefaultExt = ".png"
let mutable DefaultImgRes = new Point(1600, 1600)
let HandShake = Color.FromArgb(0xff, 0x04, 0x09, 0x56)

type AlphaFlags =
    static member ALL = 0xff // Read all
    static member END = 0x00 // Read nothing
    static member R   = 0x11 // Read red
    static member RG  = 0x22 // Read red, green

// =============================================

(* Calculate the amount of bytes the can be stored in an image of the specified resolution *)
let CalcImgBytes x y mult =
    (x * y) * mult


(* Calculate the amount of images that will be needed to store the data *)
let CalcNumOfImgsRequired (dataBytes:int) (imgBytes:int) =
    let headerSize = 16 + MaxFileNameSize + 1
    let amount = (float dataBytes) / (float imgBytes) |> ceil |> int

    if ((float (dataBytes + (amount * headerSize))) / (float imgBytes) |> ceil |> int > amount) then amount + 1
    else amount


(* Break an sequence into a sequence of array's *)
let Break n (s:seq<_>) = seq {
    use e = s.GetEnumerator()
    while e.MoveNext() do
        let i = ref 0
        yield [|
            yield e.Current
            i := !i + 1
            while !i < n && e.MoveNext() do            
                yield e.Current
                i := !i + 1 |]
    }


(* Generates the header data that contains the needed info *)
let CreateHeader part totalParts fileVersion (fileName:string) =
    let GetColor (b:byte[]) =
        match b.Length with
        | 0 -> Color.FromArgb(0x00, 0x00, 0x00, 0x00)
        | 1 -> Color.FromArgb(int b.[0], 0x00, 0x00, 0x00)
        | 2 -> Color.FromArgb(int b.[0], int b.[1], 0x00, 0x00)
        | 3 -> Color.FromArgb(int b.[0], int b.[1], int b.[2], 0x00)
        | _ -> Color.FromArgb(int b.[0], int b.[1], int b.[2], int b.[3])

    if (fileName.Length > MaxFileNameSize) then
        let msg = "File name length must be " + string MaxFileNameSize + " or smaller and bigger than 0"
        let fn = "fileName"
        raise(new ArgumentOutOfRangeException(fn, fileName.Length, msg))
    else
        [| HandShake; HandShake; HandShake |]
        |> fun colors -> Array.append colors [|Color.FromArgb(MaxFileNameSize, part, totalParts, fileVersion)|]
        |> fun colors -> (colors, Break 4 (Encoding.UTF8.GetBytes(fileName) |> Seq.ofArray))
        |> fun (colors, bytes) -> (colors, [for item in bytes -> GetColor item])
        |> fun (colors, colors') -> Array.append colors (Array.ofList colors')


(* Generates a 'usable' filename *)
let DoCorrectFileName (fileInfo:FileInfo) =
    if (fileInfo.Name.Length > MaxFileNameSize) then
        if (fileInfo.Name.Contains(".")) then
            fileInfo.Extension.ToCharArray()
            |> fun chars -> Array.append chars (fileInfo.Name.ToCharArray(0, MaxFileNameSize - (fileInfo.Extension.Length - 1)))
            |> fun chars -> new String(chars)
        else
            fileInfo.Name.ToCharArray(0, MaxFileNameSize)
            |> fun chars -> new String(chars)
    else
        fileInfo.Name.ToCharArray()
        |> fun chars -> Array.append chars [|for i in 0 .. MaxFileNameSize - chars.Length - 1 -> '\x00'|]
        |> fun chars -> new String(chars)


(* Creates ready to use data out of which we can generate our images *)
let CreateImages file targetDirectory =
    let fileInfo = new FileInfo(file)
    let fileName = DoCorrectFileName fileInfo
    let numOfImgs = CalcNumOfImgsRequired (int fileInfo.Length)  (CalcImgBytes DefaultImgRes.X DefaultImgRes.Y 3)

    [for i in [ 0 .. numOfImgs - 1 ] -> async {
        let header = CreateHeader (i+1) numOfImgs FileVersion fileName
        use bmp = new Bitmap(DefaultImgRes.X, DefaultImgRes.Y, PixelFormat.Format32bppArgb)
        use fs = fileInfo.OpenRead()

        let GetX y = if (y = 0) then header.Length - 1 else 0

        // Set file stream offset
        fs.Position <- int64 ((((bmp.Width * bmp.Height) - header.Length) * 3) * i)

        // Write header
        for x in [ 0 .. header.Length - 1 ] do bmp.SetPixel(x, 0, header.[x])

        // Bytes to pixels
        for y in [ 0 .. bmp.Height - 1 ] do
            for x in [ GetX y .. bmp.Width - 1 ] do
                match ((fs.Length - 1L) - fs.Position) with
                | 0L -> bmp.SetPixel(x, y, Color.FromArgb(AlphaFlags.END, 0x00, 0x00, 0x00))
                | 1L -> bmp.SetPixel(x, y, Color.FromArgb(AlphaFlags.R, fs.ReadByte(), 0x00, 0x00))
                | 2L -> bmp.SetPixel(x, y, Color.FromArgb(AlphaFlags.RG, fs.ReadByte(), fs.ReadByte(), 0x00))
                | _ -> bmp.SetPixel(x, y, Color.FromArgb(AlphaFlags.ALL, fs.ReadByte(), fs.ReadByte(), fs.ReadByte()))

        let savePath = (*targetDirectory + "\\" +*) DateTime.Now.Ticks.ToString("x") + DefaultExt
        bmp.Save(savePath)

        return savePath}]
    |> Async.Parallel
    |> Async.RunSynchronously


(* Creates ready to use data out of which we can generate our images *)
let CreateImages2 file targetDirectory =
    let fileInfo = new FileInfo(file)
    let fileName = DoCorrectFileName fileInfo
    let numOfImgs = CalcNumOfImgsRequired (int fileInfo.Length)  (CalcImgBytes DefaultImgRes.X DefaultImgRes.Y 3)
    let headerLength = (CreateHeader 1 numOfImgs FileVersion fileName).Length
    use fs = fileInfo.OpenRead()
    let buffers = File.ReadAllBytes(file)
                  |> Seq.ofArray
                  |> Break (((DefaultImgRes.X * DefaultImgRes.Y) - headerLength) * 3)
                  |> Array.ofSeq
                  |> fun b -> [for i in 0 .. numOfImgs - 1 -> async{return (b.[i], CreateHeader (i + 1) numOfImgs FileVersion fileName)}]
                  |> Async.Parallel
                  |> Async.RunSynchronously
    
    [for b, h in buffers -> async {
        use bmp = new Bitmap(DefaultImgRes.X, DefaultImgRes.Y, PixelFormat.Format32bppArgb)

        // implementation....

        let savePath = (*targetDirectory + "\\" +*) DateTime.Now.Ticks.ToString("x") + DefaultExt
        //bmp.Save(savePath)

        //[ savePath ]
        //|> (fun s -> List.append s savedFiles)
        //|> loop (i + 1)
        return savePath}]
    |> Async.Parallel
    |> Async.RunSynchronously


(* Creates ready to use data out of which we can generate our images *)
let CreateImages3 file targetDirectory = 
    let fileInfo = new FileInfo(file)
    let fileName = DoCorrectFileName fileInfo
    let numOfImgs = CalcNumOfImgsRequired (int fileInfo.Length)  (CalcImgBytes DefaultImgRes.X DefaultImgRes.Y 3)
    
    Parallel.For(0, numOfImgs - 1, (fun i ->
        let header = CreateHeader (i+1) numOfImgs FileVersion fileName
        use bmp = new Bitmap(DefaultImgRes.X, DefaultImgRes.Y, PixelFormat.Format32bppArgb)
        use g = Graphics.FromImage(bmp)
        use fs = fileInfo.OpenRead()

        let GetX y = if (y = 0) then header.Length - 1 else 0

        fs.Position <- int64 ((((bmp.Width * bmp.Height) - header.Length) * 3) * i)

        // Write header
        for x in [ 0 .. header.Length - 1 ] do bmp.SetPixel(x, 0, header.[x])

        //printfn "Processing Image %i of %i" (i + 1) numOfImgs

        for y in [ 0 .. DefaultImgRes.Y - 1 ] do
            for x in [ 0 .. DefaultImgRes.X - 1 ] do
                match ((fs.Length - 1L) - fs.Position) with
                | 0L -> g.DrawRectangle(new Pen(Color.FromArgb(AlphaFlags.END, 0x00, 0x00, 0x00)), x, y, 1, 1)
                | 1L -> g.DrawRectangle(new Pen(Color.FromArgb(AlphaFlags.R, fs.ReadByte(), 0x00, 0x00)), x, y, 1, 1)
                | 2L -> g.DrawRectangle(new Pen(Color.FromArgb(AlphaFlags.RG, fs.ReadByte(), fs.ReadByte(), 0x00)), x, y, 1, 1)
                | _ -> g.DrawRectangle(new Pen(Color.FromArgb(AlphaFlags.ALL, fs.ReadByte(), fs.ReadByte(), fs.ReadByte())), x, y, 1, 1)

        let savePath = (*targetDirectory + @"\" +*) DateTime.Now.Ticks.ToString("x") + DefaultExt
        
        bmp.Save(savePath)
    ))

CreateImages3() is the synchronous version, CreateImages2() is the multi core version with a RAM buffer, CreateImages() the multi core version without a RAM buffer.

All of them are slow, GDI(+) being the major bottleneck here. If someone knows of a good image manipulation library I'd like to hear it because GDI will be to slow for anyone.

I could use DirectDraw, Direct2D, DirectCompute 10 or OpenCL but all of those require to present on the client system so an alternative would be nice.

See now this is why a coding section would be handy on KWWH

[Hyperz]

Finally cracked it, the fasted encoding speed the above methods gave me was 650 Kbyte/sec. I switched to FastPixel for the drawing and added functions to FastPixel to be able to draw pixels via bytes instead of passing Color objects. Also changed to staged caching. New encoding speed: 2-3 Mbyte/sec. That is faster than the speed I get in the winrar bechmark so should be a fast enough starting point for peoples.

The magic, for those interested:

let CreateImages file =
    let fileInfo = new FileInfo(file)
    let fileName = DoCorrectFileName fileInfo
    let numOfImgs = CalcNumOfImgsRequired (int fileInfo.Length)  (CalcImgBytes DefaultImgRes.X DefaultImgRes.Y 3)

    [for i in [ 0 .. numOfImgs - 1 ] -> async {
        let header = CreateHeader (i+1) numOfImgs FileVersion fileName
        use fs = fileInfo.OpenRead()
        use bmp = new Bitmap(DefaultImgRes.X, DefaultImgRes.Y, PixelFormat.Format32bppArgb)
        let fpx = new FastPixel(bmp)

        fs.Position <- int64 ((((DefaultImgRes.X * DefaultImgRes.Y) - header.Length) * 3) * i)

        let length = ((DefaultImgRes.X * DefaultImgRes.Y) - header.Length) * 3

        let count = ref 0
        let buffer' = ref (Array.create (length + 1) 0x00uy)
        let GetX y = if (y = 0) then header.Length - 1 else 0
        let realLength = fs.Read(!buffer', 0, (!buffer').Length)
        let buffer = (!buffer').[0 .. realLength - 1]
                     |> Seq.ofArray
                     |> Break 3
                     |> Array.ofSeq

        // Set file stream offset and free temp buffer
        buffer' := Array.empty<byte>
        fpx.Lock()

        // Write header
        for x in [ 0 .. header.Length - 1 ] do fpx.SetPixel(x, 0, header.[x])
        
        // Bytes to pixels
        for y in [ 0 .. DefaultImgRes.Y - 1 ] do
            for x in [ GetX y .. DefaultImgRes.X - 1 ] do
                if (!count < buffer.Length) then
                    match (buffer.[!count].Length) with
                        | 3 -> fpx.SetBytes(x, y, byte AlphaFlags.ALL, buffer.[!count].[0], buffer.[!count].[1], buffer.[!count].[2])
                        | 2 -> fpx.SetBytes(x, y, byte AlphaFlags.RG, buffer.[!count].[0], buffer.[!count].[1], 0x00uy)
                        | 1 -> fpx.SetBytes(x, y, byte AlphaFlags.R, buffer.[!count].[0], 0x00uy, 0x00uy)
                        | _ -> fpx.SetBytes(x, y, byte AlphaFlags.END, 0x00uy, 0x00uy, 0x00uy)

                    count := !count + 1
                else
                    fpx.SetBytes(x, y, byte AlphaFlags.END, 0x00uy, 0x00uy, 0x00uy)

        fpx.Unlock(true)

        return bmp}]
    |> Async.Parallel
    |> Async.RunSynchronously

Anyway, I wanted to ask if some would be willing to run a test program that will bechmark encoding speed? It would require you to download .NET 4 Beta 2 though.

[SplitIce]

Gee I think mines much simpler, btw php encodes a 100mb file in less than 10 seconds

[Hyperz]

That is pretty slow for just passing bytes. We'll talk again about speeds when you have a working transcoder ^^ (working as being able to split data across multiple images, one that knows when it's decoding a data image instead of any image, and one that doesn't corrupt data in the whole process). You'll be lucky to get 100 kbyte/sec .

[SplitIce]

Actually mine already splits accross multiple images
CreateSplit

[Hyperz]

I demand source code to feed my XAMPP install.