reading/writing byte offset compression in cbf

Posted on by

The compression scheme “byte offset” is commonly used in frames encoded in cbf. The Pilatus hybrid pixel detectors are using this format. The algorithm is descrived here: http://www.bernstein-plus-sons.com/software/CBF/doc/CBFlib.html#3.3 and a library already exist to manipulate cbf files called cbflib. The library is very extensive but I only wanted a small piece of code to read or write cbf with byte_offset compression in Fortran. As a result I wrote my own.

Reading algorithm. position at starts is the begining of the binary section. To save I/O a character buffer is used. data are read until the number of pixels as been reached. frame%array is the destination as a 2D array.

basepixel=0
x=0
y=1
i=1
read(filehandled, pos=position, iostat=ierr) buffer
do 
    if(i>len(buffer)-16) then
        position=position+i-1
        read(filehandled, pos=position, iostat=ierr) buffer
        i=1
    end if
    c1=buffer(i:i)
    isize=1
    if(c1==char(128)) then
        isize=2
        i=i+1
        c2=buffer(i:i+1)
        if(c2==char(0)//char(128)) then
            isize=4
            i=i+2
            c4=buffer(i:i+3)
            if(c4==char(0)//char(0)//char(0)//char(128)) then            
                isize=8
                i=i+4
            end if
        end if
    end if
 
    select case(isize)
        case(1)
        ionebyte=ichar(buffer(i:i))
        i=i+1
        basepixel=basepixel+ionebyte
        case(2)
        itwobytes=ichar(buffer(i:i))
        itwobytes=itwobytes+ishft(ichar(buffer(i+1:i+1)), 8)
        i=i+2
        basepixel=basepixel+itwobytes
        case(4)
        ifourbytes=ichar(buffer(i:i))
        ifourbytes=ifourbytes+ishft(ichar(buffer(i+1:i+1)), 8)
        ifourbytes=ifourbytes+ishft(ichar(buffer(i+2:i+2)), 16)
        ifourbytes=ifourbytes+ishft(ichar(buffer(i+3:i+3)), 24)
        i=i+4
        basepixel=basepixel+ifourbytes
        case(8)
        print *, 'not supported'
        stop
        case default
        print *, 'big ooops'
        stop
    end select
 
    x=x+1
    if(x>ubound(frame%array, 1)) then
        y=y+1
        x=1
		!print *, x, y-1, basepixel
    end if
    if(y>ubound(frame%array, 2))  then
        exit
    end if
    frame%array(x,y)=basepixel
end do

Writing algorithm. Same here, to save I/O a character buffer is used. cbf is the original 2D array. scrtachfile is the temporary destination file.

basepixel=0
byteswritten=0
totalbyteswritten=0
! origin is at the bottom
do j = ubound(cbf,2), 1, -1
    do k = 1, ubound(cbf,1)
        ! if the buffer is full, write the content to the file
        if(byteswritten>buffer_length-15) then
            write(scratchfile) longbuffer(1:byteswritten)
            totalbyteswritten=totalbyteswritten+byteswritten
            !print *, totalbyteswritten, byteswritten
            byteswritten=0
        end if
 
        deltai=cbf(k,j)-basepixel
        if(deltai>=-127 .and. deltai<=127) then
            longbuffer(byteswritten+1:byteswritten+1) = char(iand(deltai, z'FF'))
            byteswritten=byteswritten+1
            basepixel=cbf(k,j)
        else if(deltai>=-32767 .and. deltai<=32767) then
            ! -128 signed == 128 unsigned
            longbuffer(byteswritten+1:byteswritten+1) = char(128)
            longbuffer(byteswritten+2:byteswritten+2) = char(iand(deltai, z'FF'))
            longbuffer(byteswritten+3:byteswritten+3) = char(iand(ishft(deltai, -8), z'FF'))
            byteswritten=byteswritten+3
            basepixel=cbf(k,j)
        else if(deltai>=-2147483647 .and. deltai<=2147483647) then
            ! 0x8000 split in 2 character and manually switch for little_endian
            longbuffer(byteswritten+1:byteswritten+3) = char(128)//char(0)//char(128) ! 0x80 and 0x8000 little_endian
            longbuffer(byteswritten+4:byteswritten+4) = char(iand(deltai, z'FF'))
            longbuffer(byteswritten+5:byteswritten+5) = char(iand(ishft(deltai, -8), z'FF'))
            longbuffer(byteswritten+6:byteswritten+6) = char(iand(ishft(deltai, -16), z'FF'))
            longbuffer(byteswritten+7:byteswritten+7) = char(iand(ishft(deltai, -24), z'FF'))
            byteswritten=byteswritten+7
            basepixel=cbf(k,j)
        else
            ! this part should work
            print *, '64bits integers not implemented!!!!'
            stop
            longbuffer(byteswritten+1:byteswritten+7) = char(128)//char(0)//&
            &   char(128)//char(0)//char(0)//char(0)//char(128)
            longbuffer(byteswritten+8:byteswritten+8) = char(iand(deltai, z'FF'))
            longbuffer(byteswritten+9:byteswritten+9) = char(iand(ishft(deltai, -8), z'FF'))
            longbuffer(byteswritten+10:byteswritten+10) = char(iand(ishft(deltai, -16), z'FF'))
            longbuffer(byteswritten+11:byteswritten+11) = char(iand(ishft(deltai, -24), z'FF'))
            longbuffer(byteswritten+12:byteswritten+12) = char(iand(ishft(deltai, -32), z'FF'))
            longbuffer(byteswritten+13:byteswritten+13) = char(iand(ishft(deltai, -40), z'FF'))
            longbuffer(byteswritten+14:byteswritten+14) = char(iand(ishft(deltai, -48), z'FF'))
            longbuffer(byteswritten+15:byteswritten+15) = char(iand(ishft(deltai, -56), z'FF'))
            byteswritten=byteswritten+15
            basepixel=cbf(k,j)
        end if
    end do
end do

One thought on “reading/writing byte offset compression in cbf”

  1. Pingback: Compression and crystallograpic 2D images | Debroglie's repository