triangle stuffs | Python Fiddle

def gcd(a,b):
    if(a==b):
      return a
    elif(a==0):
      return b
    elif(b==0):
      return a
    elif(a%2==0):
      if(b%2==0):
        return gcd(a/2, b/2)*2
      else:
         return gcd(a/2, b)
    elif(b%2==0):
      return gcd(a, b/2)
    elif(a>b):
      return gcd( (a-b)/2, b)
    else:
      return gcd( (b-a)/2, a)

def getLine(x,y):
    if(gcd(x,y)==0):
        return 1
    div = x/gcd(x,y)
    return x/div - 1

def rightTri(a,b,c):
    if(a[0]==b[0]):
        y=abs(a[1]-b[1])
        if(a[1]==c[1]):
            x=abs(a[0]-c[0])
        elif(b[1]==c[1]):
            x=abs(b[0]-c[0])
    elif(a[0]==c[0]):
        y=abs(a[1]-c[1])
        if(a[1]==b[1]):
            x=abs(a[0]-b[0])
        elif(b[1]==c[1]):
            x=abs(b[0]-c[0])
            
    elif(b[0]==c[0]):
        y=abs(b[1]-c[1])
        if(a[1]==b[1]):
            x=abs(a[0]-b[0])
        elif(a[1]==c[1]):
            x=abs(a[0]-c[0])
    return ( (x-1) * (y-1) - getLine(x,y) ) /2

def inLineTri(a,b,c):
    x0 = max(a[0], b[0], c[0])-min(a[0], b[0], c[0])
    y0 = max(a[1], b[1], c[1])-min(a[1], b[1], c[1])
    rect = (x0-1) * (y0-1)
    bigTri = rightTri([0,0],[x0,0], [0, y0])
    
    #j=x0
    #k=y0
    
    if(a[0]==b[0]):
        y1 = abs(a[1]-b[1])
        y2 = abs(y0-y1)
        return rect - getLine(x0,y2) - getLine(x0, y0) - bigTri - rightTri([0,0], [x0, 0], [0, y2])            
    elif(a[0]==c[0]):
        y1 = abs(a[1]-c[1])
        y2 = abs(y0-y1)
        return rect - getLine(x0,y2) - getLine(x0, y0) - bigTri - rightTri([0,0], [x0, 0], [0, y2])
    elif(b[0]==c[0]):
        y1 = abs(c[1]-b[1])
        y2 = abs(y0-y1)
        return rect - getLine(x0,y2) - getLine(x0, y0) - bigTri - rightTri([0,0], [x0, 0], [0, y2])                    
    elif(a[1]==b[1]):
        x1 = abs(a[1]-b[1])
        x2 = abs(x0-x1)
        return rect - getLine(x2, y0) - getLine(x0,y0) - bigTri - rightTri([0,0], [x2, 0], [0, y0])
    elif(a[1]==c[1]):
        x1 = abs(a[1]-c[1])
        x2 = abs(x0-x1)
        return rect - getLine(x2, y0) - getLine(x0,y0) - bigTri - rightTri([0,0], [x2, 0], [0, y0])
    elif(b[1]==c[1]):
        x1 = abs(b[1]-c[1])
        x2 = abs(x0-x1)
        return rect - getLine(x2, y0) - getLine(x0,y0) - bigTri - rightTri([0,0], [x2, 0], [0, y0])

def fullTri(a,b,c):
    #find min, max, and mid X
    if(a[0]<b[0] and a[0]<c[0]):
        minX=a[0]
        if(b[0]<c[0]):
            midX=b[0]
            maxX=c[0]
        else:
            midX=c[0]
            maxX=b[0]
    elif(b[0]<c[0] and b[0]<a[0]):
        minX=b[0]
        if(a[0]<c[0]):
            midX=a[0]
            maxX=c[0]
        else:
            midX=c[0]
            maxX=a[0]
            
    elif(c[0]<a[0] and c[0]<b[0]):
        minX=c[0]
        if(b[0]<a[0]):
            midX=b[0]
            maxX=a[0]
        else:
            midX=a[0]
            maxX=b[0]
            
            
    #do the same for y
    if(a[1]<b[1] and a[1]<c[1]):
        minY=a[1]
        if(b[1]<c[1]):
            midY=b[1]
            maxY=c[1]
        else:
            midY=c[1]
            maxY=b[1]
    elif(b[1]<c[1] and b[1]<a[1]):
        minY=b[1]
        if(a[1]<c[1]):
            midY=a[1]
            maxY=c[1]
        else:
            midY=c[1]
            maxY=a[1]
            
    elif(c[1]<a[1] and c[1]<b[1]):
        minY=c[1]
        if(b[1]<a[1]):
            midY=b[1]
            maxY=a[1]
        else:
            midY=a[1]
            maxY=b[1]
    
    #construct the box
    x0 = maxX-minX
    y0 = maxY-minY
    rect = (x0-1) * (y0-1)
    
   
    
    l1=getLine( midX-minX, maxY-midY)
    l2=getLine( maxX-minX, midY-minY)
    l3=getLine( maxX-midX, maxY-minY)
    smallRect = 0
    
    
    
    
    #we have a special case where two vertices are in corners of the box.
    
    
    if(a[0]==minX and a[1]==minY):
        if( (b[0]==maxX and b[1]==maxY) or (c[0]==maxX and c[1]==maxY) ):
            r1 = (maxX-midX-1) * (midY-minY-1)
            r2 = (midX-minX-1) * (maxY-midY-1)
            if(r1<r2):
                smallRect = r1
                t1= rightTri( [minX,maxY],[maxX,maxY],[minX,minY] )
                t2= rightTri( [minX,minY],[midX,midY],[midX,minY] )
                t3= rightTri( [maxX,maxY],[midX,midY],[maxX,midY] )
            else:
                smallRect = r2
                t1= [midX,maxY],[midX,midY],[maxX,maxY]
                t2= [minX,midY],[midX,midY],[minX,minY]
                t3= [minX,minY],[maxX,minY],[maxX,maxY]
    elif(a[0]==minX and a[1]==maxY):
        if( (b[0]==maxX and b[1]==minY) or (c[0]==maxX and c[1]==minY) ):
            r1 = (maxX-midX-1) * (maxY-midY-1)
            r2 = (midX-minX-1) * (midY-minY-1)
            if(r1<r2):
                smallRect = r1
                t1= rightTri( [minX,maxY],[minX,minY],[maxX,minY] )
                t2= rightTri( [minX,maxY],[midX,maxY],[midX,midY] )
                t3= rightTri( [midX,midY],[maxX,midY],[maxX,minY] )
            else:
                smallRect = r2
                t1= rightTri( [minX,maxY],[minX,midY],[midX,midY] )
                t2= rightTri( [midx,midY],[midX,minY],[maxX,minY] )
                t3= rightTri( [minx,maxY],[maxX,maxY],[maxX,minY] )
                
                
    elif(b[0]==minX and b[1]==minY):
        if( (a[0]==maxX and a[1]==maxY) or (c[0]==maxX and c[1]==maxY) ):
            r1 = (maxX-midX-1) * (midY-minY-1)
            r2 = (midX-minX-1) * (maxY-midY-1)
            if(r1<r2):
                smallRect = r1
                t1= rightTri( [minX,maxY],[maxX,maxY],[minX,minY] )
                t2= rightTri( [minX,minY],[midX,midY],[midX,minY] )
                t3= rightTri( [maxX,maxY],[midX,midY],[maxX,midY] )
            else:
                smallRect = r2
                t1= [midX,maxY],[midX,midY],[maxX,maxY]
                t2= [minX,midY],[midX,midY],[minX,minY]
                t3= [minX,minY],[maxX,minY],[maxX,maxY]
    elif(b[0]==minX and b[1]==maxY):
        if( (a[0]==maxX and a[1]==minY) or (c[0]==maxX and c[1]==minY) ):
            r1 = (maxX-midX-1) * (maxY-midY-1)
            r2 = (midX-minX-1) * (midY-minY-1)
            if(r1<r2):
                smallRect = r1
                t1= rightTri( [minX,maxY],[minX,minY],[maxX,minY] )
                t2= rightTri( [minX,maxY],[midX,maxY],[midX,midY] )
                t3= rightTri( [midX,midY],[maxX,midY],[maxX,minY] )
            else:
                smallRect = r2
                t1= rightTri( [minX,maxY],[minX,midY],[midX,midY] )
                t2= rightTri( [midx,midY],[midX,minY],[maxX,minY] )
                t3= rightTri( [minx,maxY],[maxX,maxY],[maxX,minY] )
                
                
                
    elif(c[0]==minX and c[1]==minY):
        if( (b[0]==maxX and b[1]==maxY) or (a[0]==maxX and a[1]==maxY) ):
            r1 = (maxX-midX-1) * (midY-minY-1)
            r2 = (midX-minX-1) * (maxY-midY-1)
            if(r1<r2):
                smallRect = r1
                t1= rightTri( [minX,maxY],[maxX,maxY],[minX,minY] )
                t2= rightTri( [minX,minY],[midX,midY],[midX,minY] )
                t3= rightTri( [maxX,maxY],[midX,midY],[maxX,midY] )
            else:
                smallRect = r2
                t1= [midX,maxY],[midX,midY],[maxX,maxY]
                t2= [minX,midY],[midX,midY],[minX,minY]
                t3= [minX,minY],[maxX,minY],[maxX,maxY]
    elif(c[0]==minX and c[1]==maxY):
        if( (b[0]==maxX and b[1]==minY) or (a[0]==maxX and a[1]==minY) ):
            r1 = (maxX-midX-1) * (maxY-midY-1)
            r2 = (midX-minX-1) * (midY-minY-1)
            if(r1<r2):
                smallRect = r1
                t1= rightTri( [minX,maxY],[minX,minY],[maxX,minY] )
                t2= rightTri( [minX,maxY],[midX,maxY],[midX,midY] )
                t3= rightTri( [midX,midY],[maxX,midY],[maxX,minY] )
            else:
                smallRect = r2
                t1= rightTri( [minX,maxY],[minX,midY],[midX,midY] )
                t2= rightTri( [midx,midY],[midX,minY],[maxX,minY] )
                t3= rightTri( [minx,maxY],[maxX,maxY],[maxX,minY] )
    else:
         t1= rightTri( [minX, minY], [minX, midY], [maxX, minY] )
         t2= rightTri( [minX, midY], [minX, maxY], [midX, maxY] )
         t3= rightTri( [midX, maxY], [maxX, maxY], [maxX, minY] )
            
        
        
#    print("smallRect: "+str(smallRect) +"   t1 "+str(t1) +"  t2 "+str(t2) +"  t3 "+str(t3))

return rect -l1 -l2 -l3 -t1 -t2 -t3 - smallRect
        
        
def answer(vertices):
    vertical = None
    horizontal = None
    
    if( (vertices[0][0]==vertices[1][0]) or (vertices[0][0]==vertices[2][0]) or (vertices[1][0]==vertices[2][0]) ):
        horizontal = True
    if( (vertices[0][1]==vertices[1][1]) or (vertices[0][1]==vertices[2][1]) or (vertices[1][1]==vertices[2][1]) ):
        vertical = True
        
        
    if( vertical and horizontal):
        return rightTri(vertices[0], vertices[1], vertices[2])
    elif( vertical or horizontal):
        return inLineTri(vertices[0], vertices[1], vertices[2])
    else:
        return fullTri(vertices[0], vertices[1], vertices[2])

print( answer( [[1,1], [6,1], [6,3]] ) )
print( answer( [[0,0], [0,4], [8,0]] ) )
print( answer( [[1,3], [1,7], [5,1]] ) )
print( answer( [[1,2], [3,6], [6,1]] ) )
print( answer( [[0,0], [2,1], [3,4]] ) )

print( answer([[2, 3], [6, 9], [10, 160]]) )
print( answer([[91207, 89566], [-88690, -83026], [67100, 47194]]) )

Python Fiddle

Python Cloud IDE