Spilt Handwritten Paragraphs
Software Recommendations does not render iPython notebooks so you can see the rendered, by nbviewer, version of this code here or of a cleaned up version here.
N.B. The example page in the question was written on unlined paper so the paragraphs were at all angles - this was edited so that the text was at right angles to the left border by hand. While it would be possible to add to the code here so as to group the text, detect the angles and do the rotation a much simpler solution is to use lined paper.
## For Python 2.6+ and Python 3.x compatibility
from future import print_function
## Import the tools that we will use
from scipy import ndimage, misc # Used for image manipulations
import numpy as np # The heavy lifter
import matplotlib.pyplot as plt # Mostly used for debug & illistration displays
sample = misc.imread('Sample.jpg') # Read in the image
# Lets have a display so that we know we have the image loaded OK!
plt.imshow(sample, aspect='auto') # Create a plot
ax = plt.gca() # Get the axis
ax.tick_params(length=0, labelbottom="off", labelleft="off") # Hide the ticks & labels
print(sample.shape) # Lets see the dimentions while we are at it
(2340, 1654, 3)
# Define some functions that we might need
def rgb2mono(rgb):
""" Convert image to monochrome."""
return rgb.sum(2) > 450 # Sum the 3rd (Colour) Axis and convert to True/False 450 is an arbetary number
bw = rgb2mono(sample) # Get a black and white version of the image, this is quicker to manipulate
plt.imshow(bw, cmap=plt.get_cmap('gray'), aspect='auto') # Show the black & white version
ax = plt.gca() # Get the axis so we can
ax.tick_params(length=0, labelbottom="off", labelleft="off") # Turn off the ticks & lables
print (bw.shape) # You can see that this is smaller and simpler than before
(2340, 1654)
y,x = bw.shape # Get the layout
inkyrows = bw.sum(1)!=x # Count the white pixells and if they are not ALL white there must be some ink
del bw # We have finished with the B/W image now
print (y,x,sum(inkyrows)) # Debug output
occ_rows = [n for n in xrange(len(inkyrows)) if inkyrows[n]] # Get a list of the rows with some ink
# Now we need to find the mid point of each blank section
midgaps = [(occ_rows[n]+occ_rows[n+1])//2 for n in xrange(len(occ_rows)-1) if (occ_rows[n+1]-occ_rows[n])>6] # more than 6 lines are blank
print('Gaps', ', '.join([str(x) for x in midgaps])) # Debug output
2340 1654 1898
Gaps 181, 486, 752, 776, 1098, 1432, 1571, 1789, 1916, 2016, 2158
Note that the first gap and last gap may be mid way between the first/last text and the edge of the page. Lets see:
# A redline display
img1 = plt.imshow(sample, aspect='auto') # Create a plot
img1.autoscale()
for line in midgaps:
plt.axhline(y=line, color='r')
ax = plt.gca() # Get the axis
ax.tick_params(length=0, labelbottom="off", labelleft="off") # Hide the ticks & labels
plt.show()
Looking Good so far! And they are not - we are going to have to manually add them, ok so we are going to get a blank paragraph about a 3rd of the way down but that should be acceptable.
# We need to add the first and last sections to the list
sample2 = sample.copy()
(y,x,c) = sample2.shape # reget the shape
sectionlines = [0,] # start at the top edge
sectionlines.extend(midgaps) # Add the gaps
sectionlines.append(y) # Add the bottom edge
no_sections = len(sectionlines)-1 # Get the number of sections
print (no_sections)
sections = [sample2[sectionlines[n]:sectionlines[n+1]] for n in xrange(no_sections)]
#img1 = plt.imshow(sections[-1], )#aspect='auto') # Create a plot when debugging
#img1.autoscale()
# Time to save the results
for n in xrange(no_sections):
misc.imsave("output_%02d.jpg" % n, sections[n])
The results
1
2
3
Skip the blank one
5
6
7
8
9
10
11
12
Running on Original Sample
Out of curiosity I tried running this on the original sample and it came very close, (red-lined image only shown for brevity).
Summary
I have uploaded the ipython notebook, a html version of the notebook for anybody that doesn't have ipython installed yet, a standalone python file that does the same, given the requisite libraries, and all the images to here on Dropbox. Obviously this code could be improved on, made more flexible and could be expanded to straighten lines of text automatically rather than relying on them already being straight.
Note that this solution is operating system independent, free, and should work with a number of image file formats - I have not tested it with pdf files though. It relies on python, numpy & scipy but for development and debugging I also used ipython & matplotlib but these can potentially be dispensed with.
The Updated Code
# coding: utf-8
# Spilt Handwritten Paragraphs
# ---
# This notebook is an attempt to address [this](http://softwarerecs.stackexchange.com/questions/23034/software-that-can-recognize-the-paragraph-structure-of-a-scanned-handwritten-tex/23071?noredirect=1#comment33707_23071) Software Recommendations question. While it was written for handwritten documents it should work even better for typewritten or printed documents.
#
# **N.B.** The example page in the question was written on unlined paper so the paragraphs were at all angles - this is Original.jpg but in Sampe.jpg this was edited, by hand with gimp, so that the text was at right angles to the left border. While it would be possible to add to the code here so as to group the text, detect the angles and do the rotation a **much** simpler solution is to use lined paper.
# In[1]:
# For compatibility with both Python 2.6+ & 3.x
from __future__ import print_function
## Import the tools that we will use
from scipy import ndimage, misc # Used for image manipulations
import numpy as np # The heavy lifter
import matplotlib.pyplot as plt # Mostly used for debug & illistration displays
import sys, os
# In[2]:
## iPython magic to display figures inline rather than seperate windows - Don't need this in the python file.
if 'ipykernel' in sys.argv[0]:
get_ipython().magic(u'matplotlib inline')
# In[3]:
def GetImage(filename):
""" Get the image and generate the output template."""
parts = filename.split('.') # Split on .
parts[-2] = parts[-2] + "_%02d" # add the formating to the last part but one
outname_template = '.'.join(parts) # Join it back up
image = misc.imread(filename) # Read the image
return image, outname_template # Return image and outname
# In[11]:
def ShowImage(image, redlines=None, cmap=None, caption=None):
""" Show the image with optional readlines."""
if redlines is None:
redlines = []
img1 = plt.imshow(image, #aspect='auto',
cmap=cmap) # Create a plot
img1.autoscale()
for line in redlines:
plt.axhline(y=line, color='r')
ax = plt.gca() # Get the axis
ax.tick_params(length=0, labelbottom="off", labelleft="off") # Hide the ticks & labels
ax.set_title(caption)
plt.show()
# In[5]:
# Define some functions that we might need
def rgb2mono(rgb):
""" Convert image to monochrome."""
return rgb.sum(2) > 450 # Sum the 3rd (Colour) Axis and convert to True/False 450 is an arbetary number
# In[6]:
def GetGaps(image, threshold=6):
""" Get the gaps in the image that are bigger than a given threshold."""
bw = rgb2mono(image) # Convert to black & white
y,x = bw.shape # Get the layout
inkyrows = bw.sum(1)!=x # Count the white pixels and if they are not ALL white there must be some ink
del bw # We have finished with the B/W image now
occ_rows = [n for n in xrange(len(inkyrows)) if inkyrows[n]] # Get a list of the rows with some ink
# gaps are any missing sections more than threshold lines that are blank
midgaps = [(occ_rows[n]+occ_rows[n+1])//2 for n in xrange(len(occ_rows)-1) if (occ_rows[n+1]-occ_rows[n])>threshold]
return midgaps
# In[7]:
def SplitSections(image, threshold=6, showredline=False):
""" Split the image into sections - need to add top and bottom to the list."""
(y,x,c) = image.shape
sectionlines = [0,] # start at the top edge
gaps = GetGaps(image, threshold) # Get the gaps
if showredline:
ShowImage(image, redlines=gaps, caption="Red-Line")
sectionlines.extend(gaps) # Add the gaps
sectionlines.append(y) # Add the bottom edge
no_sections = len(sectionlines)-1 # Get the number of sections
sections = [image[sectionlines[n]:sectionlines[n+1]] for n in xrange(no_sections)] # Do the split
return sections
# In[13]:
def Split_File(filename, threashold=6, show=False, showresults=False):
""" Split and save a named file."""
image, outname_base = GetImage(filename) # Get the image and the output name template
if show:
ShowImage(image, caption=filename) # For demo purposes show the orginal
sections = SplitSections(image, threashold, show) # Split it into sections
for n in xrange(len(sections)): # Index on the sections
name = outname_base % n # Make the name
misc.imsave(name, sections[n]) # Save the image
if showresults: # demo section
for n in xrange(len(sections)): # Index of sections
ShowImage(sections[n], caption="%s Section %d" % (filename, n)) # Show them
# In[15]:
def Demo(showresults=False):
""" Do the demo."""
for filename in ['Sample.jpg', 'Original.jpg']: # List of demo files
if os.path.exists(filename): # If it is there
Split_File(filename, show=True, showresults=showresults) # Process it and show the results
# In[16]:
if __name__ == "__main__":
if 'ipykernel' in sys.argv[0]:
Demo(True)
elif len(sys.argv) < 2:
print(""" Usage:\n\tSupply one or more filenames on the command line.""")
Demo()
else:
for arg in sys.argv[1:]:
print("Processing", arg)
Split_File(arg)
